home *** CD-ROM | disk | FTP | other *** search
/ Chip 2007 January, February, March & April / Chip-Cover-CD-2007-02.iso / Pakiet internetowy / Rozne / HTTrack 3.40-2 / httrack-3.40-2.exe / {app} / src / htsback.c < prev    next >
C/C++ Source or Header  |  2006-04-09  |  137KB  |  3,499 lines
  1. /* ------------------------------------------------------------ */
  2. /*
  3. HTTrack Website Copier, Offline Browser for Windows and Unix
  4. Copyright (C) Xavier Roche and other contributors
  5.  
  6. This program is free software; you can redistribute it and/or
  7. modify it under the terms of the GNU General Public License
  8. as published by the Free Software Foundation; either version 2
  9. of the License, or any later version.
  10.  
  11. This program is distributed in the hope that it will be useful,
  12. but WITHOUT ANY WARRANTY; without even the implied warranty of
  13. MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14. GNU General Public License for more details.
  15.  
  16. You should have received a copy of the GNU General Public License
  17. along with this program; if not, write to the Free Software
  18. Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
  19.  
  20.  
  21. Important notes:
  22.  
  23. - We hereby ask people using this source NOT to use it in purpose of grabbing
  24. emails addresses, or collecting any other private information on persons.
  25. This would disgrace our work, and spoil the many hours we spent on it.
  26.  
  27.  
  28. Please visit our Website: http://www.httrack.com
  29. */
  30.  
  31.  
  32. /* ------------------------------------------------------------ */
  33. /* File: httrack.c subroutines:                                 */
  34. /*       backing system (multiple socket download)              */
  35. /* Author: Xavier Roche                                         */
  36. /* ------------------------------------------------------------ */
  37.  
  38. /* Internal engine bytecode */
  39. #define HTS_INTERNAL_BYTECODE
  40.  
  41. #include "htsback.h"
  42.  
  43. /* specific definitions */
  44. #include "htsbase.h"
  45. #include "htsnet.h"
  46. #include "htsthread.h"
  47. #include <time.h>
  48. /* END specific definitions */
  49.  
  50. //#if HTS_WIN
  51. #include "htsftp.h"
  52. #if HTS_USEZLIB
  53. #include "htszlib.h"
  54. #else
  55. #error HTS_USEZLIB not defined
  56. #endif
  57. //#endif
  58.  
  59. #if HTS_WIN
  60. #ifndef __cplusplus
  61. // DOS
  62. #ifndef  _WIN32_WCE
  63. #include <process.h>    /* _beginthread, _endthread */
  64. #endif
  65. #endif
  66. #else
  67. #endif
  68.  
  69. #if HTS_USEMMS
  70. #include "htsmms.h"
  71. #endif
  72.  
  73. #undef test_flush
  74. #define test_flush if (opt->flush) { if (opt->log) { fflush(opt->log); } if (opt->errlog) { fflush(opt->errlog);  } }
  75.  
  76. #define VT_CLREOL       "\33[K"
  77.  
  78.  
  79. struct_back* back_new(int back_max) {
  80.   int i;
  81.   struct_back* sback = calloct(1, sizeof(struct_back));
  82.   sback->count = back_max;
  83.   sback->lnk = (lien_back*) calloct((back_max + 1), sizeof(lien_back));
  84.   sback->ready = inthash_new(8191);
  85.   inthash_value_is_malloc(sback->ready, 1);
  86.   // init
  87.   for(i = 0 ; i < sback->count ; i++){
  88.     sback->lnk[i].r.location = sback->lnk[i].location_buffer;
  89.     sback->lnk[i].status = -1;
  90.     sback->lnk[i].r.soc = INVALID_SOCKET;
  91.   }
  92.   return sback;
  93. }
  94.  
  95. void back_free(struct_back** sback) {
  96.   if (sback != NULL && *sback != NULL) {
  97.     if ((*sback)->lnk != NULL) {
  98.       freet((*sback)->lnk);
  99.       (*sback)->lnk = NULL;
  100.     }
  101.     if ((*sback)->ready != NULL) {
  102.       inthash_delete((inthash *)&(*sback)->ready);
  103.     }
  104.     freet(*sback);
  105.     *sback = NULL;
  106.   }
  107. }
  108.  
  109. void back_delete_all(httrackp* opt, cache_back* cache, struct_back* sback) {
  110.   if (sback != NULL) {
  111.     int i;
  112.     // delete live slots
  113.     for(i = 0 ; i < sback->count ; i++) {
  114.       back_delete(opt, cache, sback, i);
  115.     }
  116.     // delete stored slots
  117.     if (sback->ready != NULL) {
  118.       struct_inthash_enum e = inthash_enum_new((inthash)sback->ready);
  119.       inthash_chain* item;
  120.       while((item = inthash_enum_next(&e))) {
  121.         struct_back back1;
  122.         back1.count = 1;
  123.         back1.lnk = (lien_back*) item->value.ptr;
  124.         back1.ready = NULL;
  125.         back_delete(opt, cache, &back1, 0);
  126.       }
  127.     }
  128.   }
  129. }
  130.  
  131. // ---
  132. // routines de backing
  133.  
  134. static int back_index_ready(struct_back* sback, char* adr, char* fil, char* sav, int getIndex);
  135. static int back_index_fetch(struct_back* sback, char* adr, char* fil, char* sav, int getIndex);
  136.  
  137. // retourne l'index d'un lien dans un tableau de backing
  138. int back_index(struct_back* sback,char* adr,char* fil,char* sav) {
  139.   return back_index_fetch(sback, adr, fil, sav, 1);
  140. }
  141.  
  142. static int back_index_fetch(struct_back* sback, char* adr, char* fil, char* sav, int getIndex) {
  143.   lien_back* const back = sback->lnk;
  144.   const int back_max = sback->count;
  145.   int i=0;
  146.   int index=-1;
  147.   while( i < back_max ) {
  148.     if (back[i].status>=0)    // rΘception OU prΩt
  149.       if (strfield2(back[i].url_adr,adr)) {
  150.         if (strcmp(back[i].url_fil,fil)==0) {
  151.           if (index==-1)    /* first time we meet, store it */
  152.             index=i;
  153.           else if (sav != NULL && strcmp(back[i].url_sav, sav) == 0) {  /* oops, check sav too */
  154.             index=i;
  155.             return index;
  156.           }
  157.         }
  158.       }
  159.     i++;
  160.   }
  161.   // not found in fast repository - search in the storage hashtable
  162.   if (index == -1 && sav != NULL) {
  163.     index = back_index_ready(sback, adr, fil, sav, getIndex);
  164.   }
  165.   return index;
  166. }
  167.  
  168. static int back_index_ready(struct_back* sback, char* adr, char* fil, char* sav, int getIndex) {
  169.   lien_back* const back = sback->lnk;
  170.   const int back_max = sback->count;
  171.   int index=-1;
  172.   void* ptr = NULL;
  173.   if (inthash_read_pvoid((inthash)sback->ready, sav, &ptr)) {
  174.     lien_back* itemback = (lien_back*) ptr;
  175.     if (itemback != NULL) {
  176.       if (!getIndex) {
  177.         return sback->count;      // positive (but invalid) result
  178.       } else {
  179.         // move from hashtable to fast repository
  180.         int q = back_search_quick(sback);
  181.         if (q != -1) {
  182.           deletehttp(&back[q].r);               // security check
  183.           back_move(itemback, &back[q]);
  184.           inthash_remove((inthash)sback->ready, sav);  // delete item
  185.           back[q].locked = 1;   /* locked */
  186.           index = q;
  187.         }
  188.       }
  189.     }
  190.   }
  191.   return index;
  192. }
  193.  
  194. /* Put all backing entries that are ready in the storage hashtable to spare space and CPU */
  195. int back_cleanup_background(httrackp* opt,cache_back* cache,struct_back* sback) {
  196.   lien_back* const back = sback->lnk;
  197.   const int back_max = sback->count;
  198.   int nclean = 0;
  199.   int i;
  200.   for( i = 0 ;  i < back_max ; i++ ) {
  201.     // ready, not locked and suitable
  202.     if (back[i].status == 0 && back[i].locked == 0 
  203.       && back[i].url_sav[0] != '\0'
  204.       && strcmp(back[i].url_sav, BACK_ADD_TEST) != 0
  205.       && !IS_DELAYED_EXT(back[i].url_sav)
  206.       ) 
  207.     {
  208.       lien_back* itemback = calloct(1, sizeof(lien_back));
  209.       /* Security check */
  210.       int checkIndex = back_index_ready(sback, back[i].url_adr, back[i].url_fil, back[i].url_sav, 1);
  211.       if (checkIndex != -1) {
  212.         if (opt->log) {
  213.           fspc(opt->log,"warning");
  214.           fprintf(opt->log,"engine: unexpected duplicate file entry: %s%s -> %s (%d '%s') / %s%s -> %s (%d '%s')"LF,
  215.             back[checkIndex].url_adr, back[checkIndex].url_fil, back[checkIndex].url_sav, back[checkIndex].r.statuscode, back[checkIndex].r.msg,
  216.             back[i].url_adr, back[i].url_fil, back[i].url_sav, back[i].r.statuscode, back[i].r.msg
  217.             );
  218.           test_flush;
  219.         }
  220.         back_delete(NULL, NULL, sback, checkIndex);
  221. #ifdef _DEBUG
  222.         /* This should NOT happend! */
  223.         { int duplicateEntryInBacklog = 1; assertf(!duplicateEntryInBacklog); }
  224. #endif
  225.       }
  226.       back_move(&back[i], itemback);
  227.       inthash_add_pvoid((inthash)sback->ready, itemback->url_sav, itemback);
  228.       nclean++;
  229.     }
  230.   }
  231.   return nclean;
  232. }
  233.  
  234. // nombre d'entrΘes libres dans le backing
  235. int back_available(struct_back* sback) {
  236.   lien_back* const back = sback->lnk;
  237.   const int back_max = sback->count;
  238.   int i;
  239.   int nb=0;
  240.   for(i=0;i<back_max;i++)
  241.     if (back[i].status==-1)     /* libre */
  242.       nb++;
  243.   return nb;
  244. }
  245.  
  246. // retourne estimation de la taille des html et fichiers stockΘs en mΘmoire
  247. LLint back_incache(struct_back* sback) {
  248.   lien_back* const back = sback->lnk;
  249.   const int back_max = sback->count;
  250.   int i;
  251.   LLint sum=0;
  252.   for(i=0;i<back_max;i++)
  253.     if (back[i].status!=-1)
  254.       if (back[i].r.adr)       // ne comptabilier que les blocs en mΘmoire
  255.         sum+=max(back[i].r.size,back[i].r.totalsize);
  256.   // stored (ready) slots
  257.   if (sback->ready != NULL) {
  258.     struct_inthash_enum e = inthash_enum_new((inthash)sback->ready);
  259.     inthash_chain* item;
  260.     while((item = inthash_enum_next(&e))) {
  261.       lien_back* ritem = (lien_back*) item->value.ptr;
  262.       if (ritem->status!=-1)
  263.         if (ritem->r.adr)       // ne comptabilier que les blocs en mΘmoire
  264.           sum+=max(ritem->r.size,ritem->r.totalsize);
  265.     }
  266.   }
  267.   return sum;
  268. }
  269.  
  270. // retourne estimation de la taille des html et fichiers stockΘs en mΘmoire
  271. int back_done_incache(struct_back* sback) {
  272.   lien_back* const back = sback->lnk;
  273.   const int back_max = sback->count;
  274.   int i;
  275.   int n=0;
  276.   for(i=0;i<back_max;i++)
  277.     if (back[i].status==0)
  278.       n++;
  279.   // stored (ready) slots
  280.   if (sback->ready != NULL) {
  281.     struct_inthash_enum e = inthash_enum_new((inthash)sback->ready);
  282.     inthash_chain* item;
  283.     while((item = inthash_enum_next(&e))) {
  284.       lien_back* ritem = (lien_back*) item->value.ptr;
  285.       if (ritem->status==0)
  286.         n++;
  287.     }
  288.   }
  289.   return n;
  290. }
  291.  
  292.  
  293. // le lien a-t-il ΘtΘ mis en backing?
  294. HTS_INLINE int back_exist(struct_back* sback,char* adr,char* fil,char* sav) {
  295.   return (back_index_fetch(sback, adr, fil, sav, /*don't fetch*/0) >= 0);
  296. }
  297.  
  298. // nombre de sockets en tΓche de fond
  299. int back_nsoc(struct_back* sback) {
  300.   lien_back* const back = sback->lnk;
  301.   const int back_max = sback->count;
  302.   int n=0;
  303.   int i;
  304.   for(i=0;i<back_max;i++)
  305.     if (back[i].status > 0)    // only receive
  306.       n++;
  307.  
  308.   return n;
  309. }
  310. int back_nsoc_overall(struct_back* sback) {
  311.   lien_back* const back = sback->lnk;
  312.   const int back_max = sback->count;
  313.   int n=0;
  314.   int i;
  315.   for(i=0;i<back_max;i++)
  316.     if (back[i].status > 0 || back[i].status == -103)
  317.       n++;
  318.  
  319.   return n;
  320. }
  321.  
  322. // objet (lien) tΘlΘchargΘ ou transfΘrΘ depuis le cache
  323. //
  324. // fermer les paramΦtres de transfert,
  325. // et notamment vΘrifier les fichiers compressΘs (dΘcompresser), callback etc.
  326. int back_finalize(httrackp* opt,cache_back* cache,struct_back* sback,int p) {
  327.   lien_back* const back = sback->lnk;
  328.   const int back_max = sback->count;
  329.   assertf(p >= 0 && p < back_max);
  330.  
  331.     /* Store ? */
  332.     if (!back[p].finalized) {
  333.         back[p].finalized = 1;
  334.  
  335.         /* Don't store broken files */
  336.         if (back[p].r.totalsize > 0 && back[p].r.size != back[p].r.totalsize && ! opt->tolerant) {
  337.             return -1;
  338.         }
  339.  
  340.         if (
  341.             (back[p].status == 0)      // ready
  342.             &&
  343.             (back[p].r.statuscode>0)   // not internal error
  344.             ) 
  345.         {
  346.             if (!back[p].testmode) {        // not test mode
  347.                 char* state="unknown";
  348.  
  349.                 /* dΘcompression */
  350. #if HTS_USEZLIB
  351.                 if (gz_is_available && back[p].r.compressed) {
  352.                     if (back[p].r.size > 0) {
  353.                         //if ( (back[p].r.adr) && (back[p].r.size>0) ) {
  354.                         // stats
  355.                         back[p].compressed_size=back[p].r.size;
  356.                         // en mΘmoire -> passage sur disque
  357.                         if (!back[p].r.is_write) {
  358.                             back[p].tmpfile_buffer[0]='\0';
  359.                             back[p].tmpfile=tmpnam(back[p].tmpfile_buffer);
  360.                             if (back[p].tmpfile != NULL && back[p].tmpfile[0] != '\0') {
  361.                                 back[p].r.out=fopen(back[p].tmpfile,"wb");
  362.                                 if (back[p].r.out) {
  363.                                     if ((back[p].r.adr) && (back[p].r.size>0)) {
  364.                                         if (fwrite(back[p].r.adr,1,(INTsys)back[p].r.size,back[p].r.out) != back[p].r.size) {
  365.                                             back[p].r.statuscode=STATUSCODE_INVALID;
  366.                                             strcpybuff(back[p].r.msg,"Write error when decompressing");
  367.                                         }
  368.                                     } else {
  369.                                         back[p].tmpfile[0]='\0';
  370.                                         back[p].r.statuscode=STATUSCODE_INVALID;
  371.                                         strcpybuff(back[p].r.msg,"Empty compressed file");
  372.                                     }
  373.                                 } else {
  374.                                     back[p].tmpfile[0]='\0';
  375.                                     back[p].r.statuscode=STATUSCODE_INVALID;
  376.                                     strcpybuff(back[p].r.msg,"Open error when decompressing");
  377.                                 }
  378.                             }
  379.                         }
  380.                         // fermer fichier sortie
  381.                         if (back[p].r.out!=NULL) {
  382.                             fclose(back[p].r.out);
  383.                             back[p].r.out=NULL;
  384.                         }
  385.                         // dΘcompression
  386.                         if (back[p].tmpfile != NULL && back[p].tmpfile[0] != '\0') {
  387.                             if (back[p].url_sav[0]) {
  388.                                 LLint size;
  389.                                 file_notify(back[p].url_adr, back[p].url_fil, back[p].url_sav, 1, 1, back[p].r.notmodified);
  390.                                 filecreateempty(back[p].url_sav);      // filenote & co
  391.                                 if ((size = hts_zunpack(back[p].tmpfile,back[p].url_sav))>=0) {
  392.                                     back[p].r.size=back[p].r.totalsize=size;
  393.                                     // fichier -> mΘmoire
  394.                                     if (!back[p].r.is_write) {
  395.                                         deleteaddr(&back[p].r);
  396.                                         back[p].r.adr=readfile(back[p].url_sav);
  397.                                         if (!back[p].r.adr) {
  398.                                             back[p].r.statuscode=STATUSCODE_INVALID;
  399.                                             strcpybuff(back[p].r.msg,"Read error when decompressing");
  400.                                         }
  401.                                         unlink(back[p].url_sav);
  402.                                     }
  403.                                 }
  404.                             }
  405.                             /* encore that no remaining temporary file exists */
  406.                             unlink(back[p].tmpfile);
  407.                             back[p].tmpfile = NULL;
  408.                         }
  409.                         // stats
  410.                         HTS_STAT.total_packed+=back[p].compressed_size;
  411.                         HTS_STAT.total_unpacked+=back[p].r.size;
  412.                         HTS_STAT.total_packedfiles++;
  413.                         // unflag
  414.                     }
  415.                 }
  416.                 back[p].r.compressed=0;
  417. #endif
  418.  
  419.                 /* Write mode to disk */
  420.                 if (back[p].r.is_write && back[p].r.adr != NULL) {
  421.                     freet(back[p].r.adr);
  422.                     back[p].r.adr = NULL;
  423.                 }
  424.  
  425.                 /* ************************************************************************
  426.                     REAL MEDIA HACK
  427.                     Check if we have to load locally the file
  428.                 ************************************************************************ */
  429.                 if (back[p].r.statuscode == 200) {    // OK (ou 304 en backing)
  430.                     if (back[p].r.is_write) {    // Written file
  431.                         if (may_be_hypertext_mime(back[p].r.contenttype, back[p].url_fil)) {   // to parse!
  432.                             LLint sz;
  433.                             sz=fsize(back[p].url_sav);
  434.                             if (sz>0) {   // ok, exists!
  435.                                 if (sz < 8192) {   // ok, small file --> to parse!
  436.                                     FILE* fp=fopen(back[p].url_sav,"rb");
  437.                                     if (fp) {
  438.                                         back[p].r.adr=malloct((int)sz + 2);
  439.                                         if (back[p].r.adr) {
  440.                                             if (fread(back[p].r.adr,1,(INTsys)sz,fp) == sz) {
  441.                                                 back[p].r.size=sz;
  442.                                                 back[p].r.adr[sz] = '\0';
  443.                                                 back[p].r.is_write = 0;                /* not anymore a direct-to-disk file */
  444.                                             } else {
  445.                                                 freet(back[p].r.adr);
  446.                                                 back[p].r.size=0;
  447.                                                 back[p].r.adr = NULL;
  448.                                                 back[p].r.statuscode=STATUSCODE_INVALID;
  449.                                                 strcpybuff(back[p].r.msg, ".RAM read error");
  450.                                             }
  451.                                             fclose(fp);
  452.                                             fp=NULL;
  453.                                             // remove (temporary) file!
  454.                                             unlink(fconv(back[p].url_sav));
  455.                                         }
  456.                                         if (fp)
  457.                                             fclose(fp);
  458.                                     }
  459.                                 }
  460.                             }
  461.                         }
  462.                     }
  463.                 }
  464.                 /* EN OF REAL MEDIA HACK */
  465.  
  466.  
  467.                 /* Stats */
  468.                 if (cache->txt) {
  469.                     char flags[32];
  470.                     char s[256];
  471.                     time_t tt;
  472.                     struct tm* A;
  473.                     tt=time(NULL);
  474.                     A=localtime(&tt);
  475.                     if (A == NULL) {
  476.                         int localtime_returned_null=0;
  477.                         assert(localtime_returned_null);
  478.                     }
  479.                     strftime(s,250,"%H:%M:%S",A);
  480.  
  481.                     flags[0]='\0';
  482.                     /* input flags */
  483.                     if (back[p].is_update)
  484.                         strcatbuff(flags, "U");   // update request
  485.                     else
  486.                         strcatbuff(flags, "-");
  487.                     if (back[p].range_req_size)
  488.                         strcatbuff(flags, "R");   // range request
  489.                     else
  490.                         strcatbuff(flags, "-");
  491.                     /* state flags */
  492.                     if (back[p].r.is_file)  // direct to disk
  493.                         strcatbuff(flags, "F");
  494.                     else
  495.                         strcatbuff(flags, "-");
  496.                     /* output flags */
  497.                     if (!back[p].r.notmodified)
  498.                         strcatbuff(flags, "M");   // modified
  499.                     else
  500.                         strcatbuff(flags, "-");
  501.                     if (back[p].r.is_chunk)  // chunked
  502.                         strcatbuff(flags, "C");
  503.                     else
  504.                         strcatbuff(flags, "-");
  505.                     if (back[p].r.compressed)
  506.                         strcatbuff(flags, "Z");   // gzip
  507.                     else
  508.                         strcatbuff(flags, "-");
  509.                     /* Err I had to split these.. */
  510.                     fprintf(cache->txt,"%s\t", s);
  511.                     fprintf(cache->txt,LLintP"/", (LLint)back[p].r.size);
  512.                     fprintf(cache->txt,LLintP,(LLint)back[p].r.totalsize);
  513.                     fprintf(cache->txt,"\t%s\t",flags);
  514.                 }
  515.                 if (back[p].r.statuscode == 200) {
  516.                     if (back[p].r.size>=0) {
  517.                         if (strcmp(back[p].url_fil,"/robots.txt") !=0 ) {
  518.                             HTS_STAT.stat_bytes+=back[p].r.size;
  519.                             HTS_STAT.stat_files++;
  520.                         }
  521.                         if ( (!back[p].r.notmodified) && (opt->is_update) ) { 
  522.                             HTS_STAT.stat_updated_files++;       // page modifiΘe
  523.                             if (opt->log!=NULL) {
  524.                                 fspc(opt->log,"info");
  525.                                 if (back[p].is_update) {
  526.                                     fprintf(opt->log,"engine: transfer-status: link updated: %s%s -> %s"LF,back[p].url_adr,back[p].url_fil,back[p].url_sav);
  527.                                 } else {
  528.                                     fprintf(opt->log,"engine: transfer-status: link added: %s%s -> %s"LF,back[p].url_adr,back[p].url_fil,back[p].url_sav);
  529.                                 }
  530.                                 test_flush;
  531.                             }
  532.                             if (cache->txt) {
  533.                                 if (back[p].is_update) {
  534.                                     state="updated";
  535.                                 } else {
  536.                                     state="added";
  537.                                 }
  538.                             }
  539.                         } else {
  540.                             if ( (opt->debug>0) && (opt->log!=NULL) ) {
  541.                                 fspc(opt->log,"info"); fprintf(opt->log,"engine: transfer-status: link recorded: %s%s -> %s"LF,back[p].url_adr,back[p].url_fil,back[p].url_sav);
  542.                                 test_flush;
  543.                             }
  544.                             if (cache->txt) {
  545.                                 if (opt->is_update)
  546.                                     state="untouched";
  547.                                 else
  548.                                     state="added";
  549.                             }
  550.                         }
  551.                     } else {
  552.                         if ( (opt->debug>0) && (opt->log!=NULL) ) {
  553.                             fspc(opt->log,"info"); fprintf(opt->log,"engine: transfer-status: empty file? (%d, '%s'): %s%s"LF,back[p].r.statuscode,back[p].r.msg,back[p].url_adr,back[p].url_fil);
  554.                             test_flush;
  555.                         }
  556.                         if (cache->txt) {
  557.                             state="empty";
  558.                         }
  559.                     }
  560.                 } else {
  561.                     if ( (opt->debug>0) && (opt->log!=NULL) ) {
  562.                         fspc(opt->log,"info"); fprintf(opt->log,"engine: transfer-status: link error (%d, '%s'): %s%s"LF,back[p].r.statuscode,back[p].r.msg,back[p].url_adr,back[p].url_fil);
  563.                     }
  564.                     if (cache->txt) {
  565.                         state="error";
  566.                     }
  567.                 }
  568.                 if (cache->txt) {
  569.                     fprintf(cache->txt,
  570.                         "%d\t"
  571.                         "%s ('%s')\t"
  572.                         "%s\t"
  573.                         "%s%s\t"
  574.                         "%s%s%s\t%s\t"
  575.                         "(from %s%s%s)"
  576.                         LF,
  577.                         back[p].r.statuscode,
  578.                         state, escape_check_url_addr(back[p].r.msg),
  579.                         escape_check_url_addr(back[p].r.contenttype),
  580.                         ((back[p].r.etag[0])?"etag:":((back[p].r.lastmodified[0])?"date:":"")), escape_check_url_addr((back[p].r.etag[0])?back[p].r.etag:(back[p].r.lastmodified)),
  581.                         (link_has_authority(back[p].url_adr) ? "" : "http://"),escape_check_url_addr(back[p].url_adr),escape_check_url_addr(back[p].url_fil),escape_check_url_addr(back[p].url_sav),
  582.                         (link_has_authority(back[p].referer_adr) || !back[p].referer_adr[0]) ? "" : "http://",escape_check_url_addr(back[p].referer_adr),escape_check_url_addr(back[p].referer_fil)
  583.                         );
  584.                     if (opt->flush)
  585.                         fflush(cache->txt);
  586.                 }
  587.  
  588.                 /* Cache */
  589.                 if (!IS_DELAYED_EXT(back[p].url_sav)) {
  590.                     cache_mayadd(opt,cache,&back[p].r,back[p].url_adr,back[p].url_fil,back[p].url_sav);
  591.                 } else {
  592.                     if (!HTTP_IS_OK(back[p].r.statuscode)) {
  593.                         if ( (opt->debug>0) && (opt->log!=NULL) ) {
  594.                             fspc(opt->log,"debug"); fprintf(opt->log,"redirect to %s%s"LF,back[p].url_adr,back[p].url_fil);
  595.                         }
  596.                         /* Store only header reference */
  597.                         cache_mayadd(opt,cache,&back[p].r,back[p].url_adr,back[p].url_fil,NULL);
  598.                     } else {
  599.                         if (opt->log!=NULL) {
  600.                             fspc(opt->log,"warning"); fprintf(opt->log,"file not stored in cache due to bogus state (incomplete type): %s%s"LF,back[p].url_adr,back[p].url_fil);
  601.                         }
  602.                     }
  603.                 }
  604.  
  605.                 // status finished callback
  606. #if HTS_ANALYSTE
  607.                 if (hts_htmlcheck_xfrstatus != NULL) {
  608.                     hts_htmlcheck_xfrstatus(&back[p]);
  609.                 }
  610. #endif
  611.                 return 0;
  612.             } else {        // testmode
  613.                 if (back[p].r.statuscode / 100 >= 3) {        /* Store 3XX, 4XX, 5XX test response codes, but NOT 2XX */
  614.                     /* Cache */
  615.                     cache_mayadd(opt,cache,&back[p].r,back[p].url_adr,back[p].url_fil,NULL);
  616.                 }
  617.             }
  618.         }
  619.     }
  620.     return -1;
  621. }
  622.  
  623. /* try to keep the connection alive */
  624. int back_letlive(httrackp* opt, cache_back* cache, struct_back* sback, int p) {
  625.     lien_back* const back = sback->lnk;
  626.     const int back_max = sback->count;
  627.     int checkerror;
  628.     htsblk* src = &back[p].r;
  629.     assertf(p >= 0 && p < back_max);
  630.     if (src && !src->is_file 
  631.         && src->soc != INVALID_SOCKET
  632.         && src->statuscode >= 0           /* no timeout errors & co */
  633.         && src->keep_alive_trailers == 0  /* not yet supported (chunk trailers) */
  634.         && ! ( checkerror = check_sockerror(src->soc) )
  635.         /*&& !check_sockdata(src->soc)*/     /* no unexpected data */
  636.         ) {
  637.             htsblk tmp;
  638.             memset(&tmp, 0, sizeof(tmp));
  639.             /* clear everything but connection: switch, close, and reswitch */
  640.             back_connxfr(src, &tmp);
  641.             back_delete(opt, cache, sback, p);
  642.             //deletehttp(src);
  643.             back_connxfr(&tmp, src);
  644.             src->req.flush_garbage=1;     /* ignore CRLF garbage */
  645.             return 1;
  646.         }
  647.         return 0;
  648. }
  649.  
  650. void back_connxfr(htsblk* src, htsblk* dst) {
  651.   dst->soc = src->soc;
  652.   src->soc = INVALID_SOCKET;
  653. #if HTS_USEOPENSSL
  654.   dst->ssl = src->ssl;
  655.   src->ssl = 0;
  656.   dst->ssl_con = src->ssl_con;
  657.   src->ssl_con = NULL;
  658. #endif
  659.   dst->keep_alive = src->keep_alive;
  660.   src->keep_alive = 0;
  661.   dst->keep_alive_max = src->keep_alive_max;
  662.   src->keep_alive_max = 0;
  663.   dst->keep_alive_t = src->keep_alive_t;
  664.   src->keep_alive_t = 0;
  665.   dst->debugid = src->debugid;
  666.   src->debugid = 0;
  667. }
  668.  
  669. void back_move(lien_back* src, lien_back* dst) {
  670.   memcpy(dst, src, sizeof(lien_back));
  671.   memset(src, 0, sizeof(lien_back));  
  672.   src->r.soc=INVALID_SOCKET;
  673.   src->status=-1;
  674.   src->r.location = src->location_buffer;
  675.   dst->r.location = dst->location_buffer;
  676. }
  677.  
  678. void back_copy_static(const lien_back* src, lien_back* dst) {
  679.   memcpy(dst, src, sizeof(lien_back));
  680.   dst->r.soc=INVALID_SOCKET;
  681.     dst->r.adr = NULL;
  682.     dst->r.headers = NULL;
  683.     dst->r.out = NULL;
  684.   dst->r.location = dst->location_buffer;
  685.     dst->r.fp = NULL;
  686. #if HTS_USEOPENSSL
  687.     dst->r.ssl_con = NULL;
  688. #endif
  689. }
  690.  
  691. // clear, or leave for keep-alive
  692. int back_maydelete(httrackp* opt,cache_back* cache,struct_back* sback, int p) {
  693.   lien_back* const back = sback->lnk;
  694.   const int back_max = sback->count;
  695.   assertf(p >= 0 && p < back_max);
  696.   if (p >= 0 && p < back_max) {    // on sait jamais..
  697.     if (
  698.       /* Keep-alive authorized by user */
  699.       !opt->nokeepalive
  700.       /* Socket currently is keep-alive! */
  701.       && back[p].r.keep_alive 
  702.       /* Remaining authorized requests */
  703.       && back[p].r.keep_alive_max > 1
  704.       /* Known keep-alive start (security) */
  705.       && back[p].ka_time_start 
  706.       /* We're on time */
  707.       && time_local() < back[p].ka_time_start + back[p].r.keep_alive_t
  708.       /* Connection delay must not exceed keep-alive timeout */
  709.       && ( opt->maxconn <= 0 || ( back[p].r.keep_alive_t > ( 1.0 / opt->maxconn ) ) )
  710.       ) {
  711.       lien_back tmp;
  712.       strcpybuff(tmp.url_adr, back[p].url_adr);
  713.       if (back_letlive(opt, cache, sback, p)) {
  714.         strcpybuff(back[p].url_adr, tmp.url_adr);
  715.         back[p].status = -103;  // alive & waiting
  716.         if ((opt->debug>1) && (opt->log!=NULL)) {
  717.           fspc(opt->log,"debug"); fprintf(opt->log,"(Keep-Alive): successfully saved #%d (%s)"LF, 
  718.             back[p].r.debugid,
  719.             back[p].url_adr); test_flush;
  720.         }
  721.         return 1;
  722.       }
  723.     }
  724.     back_delete(opt,cache,sback, p);
  725.   }
  726.   return 0;
  727. }
  728.  
  729. // clear, or leave for keep-alive
  730. void back_maydeletehttp(httrackp* opt, cache_back* cache, struct_back* sback, int p) {
  731.   lien_back* const back = sback->lnk;
  732.   const int back_max = sback->count;
  733.   TStamp lt = 0;
  734.   assertf(p >= 0 && p < back_max);
  735.   if (back[p].r.soc!=INVALID_SOCKET) {
  736.     int q;
  737.     if (
  738.       back[p].r.soc != INVALID_SOCKET        /* security check */
  739.       && back[p].r.statuscode >= 0           /* no timeout errors & co */
  740.       && back[p].r.keep_alive_trailers == 0  /* not yet supported (chunk trailers) */
  741.       /* Socket not in I/O error status */
  742.       && !back[p].r.is_file
  743.       && !check_sockerror(back[p].r.soc)
  744.       /* Keep-alive authorized by user */
  745.       && !opt->nokeepalive
  746.       /* Socket currently is keep-alive! */
  747.       && back[p].r.keep_alive 
  748.       /* Remaining authorized requests */
  749.       && back[p].r.keep_alive_max > 1
  750.       /* Known keep-alive start (security) */
  751.       && back[p].ka_time_start 
  752.       /* We're on time */
  753.       && ( lt = time_local() ) < back[p].ka_time_start + back[p].r.keep_alive_t
  754.       /* Connection delay must not exceed keep-alive timeout */
  755.       && ( opt->maxconn <= 0 || ( back[p].r.keep_alive_t > ( 1.0 / opt->maxconn ) ) )
  756.       /* Available slot in backing */
  757.       && ( q = back_search(opt, cache, sback) ) >= 0
  758.       ) 
  759.     {
  760.       lien_back tmp;
  761.       strcpybuff(tmp.url_adr, back[p].url_adr);
  762.       deletehttp(&back[q].r);               // security check
  763.       back_connxfr(&back[p].r, &back[q].r); // transfer live connection settings from p to q
  764.       back[q].ka_time_start = back[p].ka_time_start;  // refresh
  765.       back[p].r.soc = INVALID_SOCKET;
  766.       strcpybuff(back[q].url_adr, tmp.url_adr); // address
  767.       back[q].status = -103;  // alive & waiting
  768.       if ((opt->debug>1) && (opt->log!=NULL)) {
  769.         fspc(opt->log,"debug"); fprintf(opt->log,"(Keep-Alive): successfully preserved #%d (%s)"LF, 
  770.           back[q].r.debugid,
  771.           back[q].url_adr); test_flush;
  772.       }
  773.     } else {
  774.       deletehttp(&back[p].r);
  775.       back[p].r.soc = INVALID_SOCKET;
  776.     }
  777.   }
  778. }
  779.  
  780.  
  781. /* attempt to attach a live connection to this slot */
  782. int back_trylive(httrackp* opt,cache_back* cache,struct_back* sback, int p) {
  783.   lien_back* const back = sback->lnk;
  784.   const int back_max = sback->count;
  785.   assertf(p >= 0 && p < back_max);
  786.   if (p>=0 && back[p].status != -103) {     // we never know..
  787.     int i = back_searchlive(opt,sback, back[p].url_adr);   // search slot
  788.     if (i >= 0 && i != p) {
  789.       deletehttp(&back[p].r);               // security check
  790.       back_connxfr(&back[i].r, &back[p].r); // transfer live connection settings from i to p
  791.       back_delete(opt,cache,sback, i);                 // delete old slot
  792.       back[p].status=100;                   // ready to connect
  793.       return 1;                             // success: will reuse live connection
  794.     }
  795.   }
  796.   return 0;
  797. }
  798.  
  799. /* search for a live position, or, if not possible, try to return a new one */
  800. int back_searchlive(httrackp* opt, struct_back* sback, char* search_addr) {
  801.   lien_back* const back = sback->lnk;
  802.   const int back_max = sback->count;
  803.   int i;
  804.  
  805.   /* search for a live socket */
  806.   for(i = 0 ; i < back_max ; i++ ) {
  807.     if (back[i].status == -103) {
  808.       if (strfield2(back[i].url_adr, search_addr)) {   /* same location (xxc: check also virtual hosts?) */
  809.         if (time_local() < back[i].ka_time_start + back[i].r.keep_alive_t) {
  810.           return i;
  811.         }
  812.       }
  813.     }
  814.   }
  815.   return -1;
  816. }
  817.   
  818. int back_search_quick(struct_back* sback) {
  819.   lien_back* const back = sback->lnk;
  820.   const int back_max = sback->count;
  821.   int i;
  822.  
  823.   /* try to find an empty place */
  824.   for(i = 0 ; i < back_max ; i++ ) {
  825.     if (back[i].status == -1) {
  826.       return i;
  827.     }
  828.   }
  829.  
  830.   /* oops, can't find a place */
  831.   return -1;
  832. }
  833.  
  834. int back_search(httrackp* opt,cache_back* cache,struct_back* sback) {
  835.   lien_back* const back = sback->lnk;
  836.   const int back_max = sback->count;
  837.   int i;
  838.  
  839.   /* try to find an empty place */
  840.   if ( ( i = back_search_quick(sback) ) != -1)
  841.     return i;
  842.  
  843.   /* couldn't find an empty place, try to requisition a keep-alive place */
  844.   for(i = 0 ; i < back_max ; i++ ) {
  845.     if (back[i].status == -103) {
  846.       /* close this place */
  847.       back_delete(opt,cache,sback, i);
  848.       return i;
  849.     }
  850.   }
  851.  
  852.   /* oops, can't find a place */
  853.   return -1;
  854. }
  855.  
  856. void back_set_finished(struct_back* sback, int p) {
  857.   lien_back* const back = sback->lnk;
  858.   const int back_max = sback->count;
  859.   assertf(p >= 0 && p < back_max);
  860.   if (p >= 0 && p < sback->count) {    // we never know..
  861.     /* status: finished (waiting to be validated) */
  862.     back[p].status=0;     /* finished */
  863.     /* close open r/w streams, if any */
  864.     if (back[p].r.fp!=NULL) {
  865.       fclose(back[p].r.fp);
  866.       back[p].r.fp=NULL;
  867.     }
  868.     if (back[p].r.out!=NULL) {  // fermer fichier sortie
  869.       fclose(back[p].r.out);
  870.       back[p].r.out=NULL;
  871.     }
  872.   }
  873. }
  874.  
  875. int back_flush_output(httrackp* opt, cache_back* cache, struct_back* sback, int p) {
  876.   lien_back* const back = sback->lnk;
  877.   const int back_max = sback->count;
  878.   assertf(p >= 0 && p < back_max);
  879.   if (p >= 0 && p < sback->count) {    // on sait jamais..
  880.         /* close input file */
  881.         if (back[p].r.fp!=NULL) {
  882.       fclose(back[p].r.fp);
  883.       back[p].r.fp=NULL;
  884.     }
  885.     /* fichier de sortie */
  886.     if (back[p].r.out!=NULL) {  // fermer fichier sortie
  887.       fclose(back[p].r.out);
  888.       back[p].r.out=NULL;
  889.     }
  890.         /* set file time */
  891.     if (back[p].r.is_write) {     // ecriture directe
  892.       /* Θcrire date "remote" */
  893.       if (strnotempty(back[p].url_sav)
  894.                 && strnotempty(back[p].r.lastmodified)
  895.                 && fexist(back[p].url_sav))          // normalement existe si on a un fichier de sortie
  896.             {
  897.         set_filetime_rfc822(back[p].url_sav,back[p].r.lastmodified);
  898.             }
  899.       /* executer commande utilisateur aprΦs chargement du fichier */
  900.       //xx usercommand(opt,0,NULL,back[p].url_sav, back[p].url_adr, back[p].url_fil);
  901.       back[p].r.is_write=0;
  902.     }
  903.         return 1;
  904.     }
  905.     return 0;
  906. }
  907.  
  908. // effacer entrΘe
  909. int back_set_passe2_ptr(httrackp* opt, cache_back* cache, struct_back* sback, int p, int* pass2_ptr) {
  910.   lien_back* const back = sback->lnk;
  911.   const int back_max = sback->count;
  912.   assertf(p >= 0 && p < back_max);
  913.   if (p >= 0 && p < sback->count) {    // on sait jamais..
  914.         back[p].pass2_ptr = pass2_ptr;
  915.       return 1;
  916.     }
  917.   return 0;
  918. }
  919.  
  920. // effacer entrΘe
  921. int back_delete(httrackp* opt, cache_back* cache, struct_back* sback, int p) {
  922.   lien_back* const back = sback->lnk;
  923.   const int back_max = sback->count;
  924.   assertf(p >= 0 && p < back_max);
  925.   if (p >= 0 && p < sback->count) {    // on sait jamais..
  926.     // VΘrificateur d'intΘgritΘ
  927. #if DEBUG_CHECKINT
  928.     _CHECKINT(&back[p],"Appel back_delete")
  929. #endif
  930. #if HTS_DEBUG_CLOSESOCK
  931.       DEBUG_W("back_delete: #%d\n" _ (int) p);
  932. #endif
  933.  
  934.     // Finalize
  935.     if (!back[p].finalized) {
  936.       if (
  937.         (back[p].status == 0)      // ready
  938.         &&
  939.         (!back[p].testmode)        // not test mode
  940.         &&
  941.         (back[p].r.statuscode>0)   // not internal error
  942.         ) {
  943.           if (opt != NULL && opt->debug>1 && opt->log!=NULL) {
  944.             fspc(opt->log,"debug"); fprintf(opt->log,"File '%s%s' -> %s not yet saved in cache - saving now"LF, back[p].url_adr, back[p].url_fil, back[p].url_sav); test_flush;
  945.           }
  946.         }
  947.         if (cache != NULL) {
  948.           back_finalize(opt, cache, sback, p);
  949.         }
  950.     }
  951.     back[p].finalized = 0;
  952.  
  953.         // flush output buffers
  954.         (void) back_flush_output(opt, cache, sback, p);
  955.     
  956.     // LibΘrer tous les sockets, handles, buffers..
  957.     if (back[p].r.soc!=INVALID_SOCKET) {
  958. #if HTS_DEBUG_CLOSESOCK
  959.       DEBUG_W("back_delete: deletehttp\n");
  960. #endif
  961.       deletehttp(&back[p].r);
  962.       back[p].r.soc=INVALID_SOCKET;
  963.     }
  964.     
  965.     if (back[p].r.adr!=NULL) {  // reste un bloc α dΘsallouer
  966.       freet(back[p].r.adr);
  967.       back[p].r.adr=NULL;
  968.     }
  969.     if (back[p].chunk_adr!=NULL) {  // reste un bloc α dΘsallouer
  970.       freet(back[p].chunk_adr);
  971.       back[p].chunk_adr=NULL;
  972.       back[p].chunk_size=0;
  973.       back[p].chunk_blocksize=0;
  974.       back[p].is_chunk=0;
  975.     }
  976.  
  977.         // only for security
  978.         if (back[p].tmpfile && back[p].tmpfile[0] != '\0') {
  979.             (void) unlink(back[p].tmpfile);
  980.             back[p].tmpfile = NULL;
  981.         }
  982.  
  983.     // headers
  984.     if (back[p].r.headers != NULL) {
  985.       freet(back[p].r.headers);
  986.       back[p].r.headers = NULL;
  987.     }
  988.  
  989.     // Tout nettoyer
  990.     memset(&back[p], 0, sizeof(lien_back));  
  991.     back[p].r.soc=INVALID_SOCKET; back[p].r.location=back[p].location_buffer;
  992.     
  993.     // Le plus important: libΘrer le champ
  994.     back[p].status=-1;
  995.       return 1;
  996.   }
  997.   return 0;
  998. }
  999.  
  1000. /* Space left on backing stack */
  1001. int back_stack_available(struct_back* sback) {
  1002.   lien_back* const back = sback->lnk;
  1003.   const int back_max = sback->count;
  1004.   int p=0,n=0;
  1005.   for( ; p < back_max ; p++ )
  1006.     if ( back[p].status == -1 )
  1007.       n++;
  1008.   return n;
  1009. }
  1010.  
  1011. // ajouter un lien en backing
  1012. int back_add_if_not_exists(struct_back* sback,httrackp* opt,cache_back* cache,char* adr,char* fil,char* save,char* referer_adr,char* referer_fil,int test,int* pass2_ptr) {
  1013.     int index = back_index(sback, adr, fil, save);
  1014.     if (index  < 0) {
  1015.     return back_add(sback, opt, cache, adr, fil, save, referer_adr, referer_fil, test, pass2_ptr);
  1016.     } else {
  1017.         /* Ensure that the reference to pass2_ptr is set */
  1018.         return back_set_passe2_ptr(opt,cache,sback,index,pass2_ptr);
  1019.     }
  1020. }
  1021.  
  1022. int back_add(struct_back* sback,httrackp* opt,cache_back* cache,char* adr,char* fil,char* save,char* referer_adr,char* referer_fil,int test,int* pass2_ptr) {
  1023.   lien_back* const back = sback->lnk;
  1024.   const int back_max = sback->count;
  1025.   int p=0;
  1026.  
  1027.   // vΘrifier cohΘrence de adr et fil (non vide!)
  1028.   if (strnotempty(adr)==0) {
  1029.     if ((opt->debug>1) && (opt->errlog!=NULL)) {
  1030.       fspc(opt->errlog,"debug"); fprintf(opt->errlog,"error: adr is empty for back_add"LF);
  1031.     }
  1032.     return -1;    // erreur!
  1033.   }
  1034.   if (strnotempty(fil)==0) {
  1035.     if ((opt->debug>1) && (opt->errlog!=NULL)) {
  1036.       fspc(opt->errlog,"debug"); fprintf(opt->errlog,"error: fil is empty for back_add"LF);
  1037.     }
  1038.     return -1;    // erreur!
  1039.   }
  1040.   // FIN vΘrifier cohΘrence de adr et fil (non vide!)
  1041.  
  1042.   // stats
  1043.   opt->state.back_add_stats++;
  1044.  
  1045.   // rechercher emplacement
  1046.   back_clean(opt, cache, sback);
  1047.   if ( ( p = back_search(opt, cache, sback) ) >= 0) {
  1048.     back[p].send_too[0]='\0';  // Θventuels paramΦtres supplΘmentaires α transmettre au serveur
  1049.  
  1050.     // clear r
  1051.     if (back[p].r.soc!=INVALID_SOCKET) {  /* we never know */
  1052.       deletehttp(&back[p].r);
  1053.     }
  1054.     memset(&(back[p].r), 0, sizeof(htsblk)); 
  1055.     back[p].r.soc=INVALID_SOCKET; 
  1056.     back[p].r.location=back[p].location_buffer;
  1057.  
  1058.     // crΘer entrΘe
  1059.     strcpybuff(back[p].url_adr,adr);
  1060.     strcpybuff(back[p].url_fil,fil);
  1061.     strcpybuff(back[p].url_sav,save);
  1062.     back[p].pass2_ptr=pass2_ptr;
  1063.     // copier referer si besoin
  1064.     strcpybuff(back[p].referer_adr,"");
  1065.     strcpybuff(back[p].referer_fil,"");
  1066.     if ((referer_adr) && (referer_fil)) {       // existe
  1067.       if ((strnotempty(referer_adr)) && (strnotempty(referer_fil))) {   // non vide
  1068.         if (referer_adr[0]!='!') {    // non dΘtruit
  1069.           if (strcmp(referer_adr,"file://")) {      // PAS file://
  1070.             if (strcmp(referer_adr,"primary")) {      // pas referer 1er lien
  1071.               strcpybuff(back[p].referer_adr,referer_adr);
  1072.               strcpybuff(back[p].referer_fil,referer_fil);
  1073.             }
  1074.           }
  1075.         }
  1076.       }
  1077.     }
  1078.     // sav ne sert α rien pour le moment
  1079.     back[p].r.size=0;                   // rien n'a encore ΘtΘ chargΘ
  1080.     back[p].r.adr=NULL;                 // pas de bloc de mΘmoire
  1081.     back[p].r.is_write=0;               // α priori stockage en mΘmoire
  1082.     back[p].maxfile_html=opt->maxfile_html;
  1083.     back[p].maxfile_nonhtml=opt->maxfile_nonhtml;
  1084.     back[p].testmode=test;              // mode test?
  1085.     if (!opt->http10)                 // option "forcer 1.0" dΘsactivΘe
  1086.       back[p].http11=1;               // autoriser http/1.1
  1087.     back[p].head_request=0;
  1088.     if (strcmp(back[p].url_sav,BACK_ADD_TEST)==0)    // HEAD
  1089.       back[p].head_request=1;
  1090.     else if (strcmp(back[p].url_sav,BACK_ADD_TEST2)==0)    // test en GET
  1091.       back[p].head_request=2;       // test en get
  1092.  
  1093.     /* Stop requested - abort backing */
  1094.     if (opt->state.stop) {
  1095.       back[p].r.statuscode=STATUSCODE_INVALID;        // fatal
  1096.       strcpybuff(back[p].r.msg,"mirror stopped by user");
  1097.       back[p].status=0;  // terminΘ
  1098.       back_set_finished(sback, p);
  1099.       if ((opt->debug>0) && (opt->log!=NULL)) {
  1100.         fspc(opt->log,"warning"); fprintf(opt->log,"File not added due to mirror cancel: %s%s"LF,adr,fil); test_flush;
  1101.       }            
  1102.       return 0;
  1103.     }
  1104.  
  1105.     // test "fast header" cache ; that is, tests we did that lead to 3XX/4XX/5XX response codes
  1106.     if (cache->cached_tests != NULL) {
  1107.       long int ptr = 0;
  1108.       if (inthash_read((inthash)cache->cached_tests, concat(adr, fil), (long int*)&ptr)) {    // gotcha
  1109.         if (ptr != 0) {
  1110.           char* text = (char*) ptr;
  1111.           char* lf = strchr(text, '\n');
  1112.           int code = 0;
  1113.           if (sscanf(text, "%d", &code) == 1) {     // got code
  1114.              back[p].r.statuscode=code;
  1115.              if (lf != NULL && *lf != '\0') {     // got location ?
  1116.                strcpybuff(back[p].r.location, lf + 1);
  1117.              }
  1118.              return 0;
  1119.           }
  1120.         }
  1121.       }
  1122.     }
  1123.  
  1124.     // tester cache
  1125.     if ((strcmp(adr,"file://"))           /* pas fichier */
  1126.       && ( (!test) || (cache->type==1) )   /* cache prioritaire, laisser passer en test! */
  1127.       && ( (strnotempty(save)) || (strcmp(fil,"/robots.txt")==0) ) ) {  // si en test on ne doit pas utiliser le cache sinon telescopage avec le 302..
  1128. #if HTS_FAST_CACHE
  1129.       long int hash_pos;
  1130.       int hash_pos_return=0;
  1131. #else
  1132.       char* a=NULL;
  1133. #endif
  1134. #if HTS_FAST_CACHE
  1135.       if (cache->hashtable) { 
  1136. #else
  1137.       if (cache->use) { 
  1138. #endif
  1139.         char BIGSTK buff[HTS_URLMAXSIZE*4];
  1140. #if HTS_FAST_CACHE
  1141.         strcpybuff(buff,adr); strcatbuff(buff,fil);
  1142.         hash_pos_return=inthash_read((inthash)cache->hashtable,buff,(long int*)&hash_pos);
  1143. #else
  1144.         buff[0]='\0'; strcatbuff(buff,"\n"); strcatbuff(buff,adr); strcatbuff(buff,"\n"); strcatbuff(buff,fil); strcatbuff(buff,"\n");
  1145.         a=strstr(cache->use,buff);
  1146. #endif
  1147.         
  1148.         // Ok, notΘ en cache->. mais bien prΘsent dans le cache ou sur disque?
  1149. #if HTS_FAST_CACHE
  1150.         if (hash_pos_return) {
  1151. #else
  1152.         if (a) {
  1153. #endif
  1154.           if (!test) {      // non mode test
  1155. #if HTS_FAST_CACHE
  1156.             int pos=hash_pos;
  1157. #else
  1158.             int pos=-1;
  1159.             a+=strlen(buff);
  1160.             sscanf(a,"%d",&pos);    // lire position
  1161. #endif
  1162.             if (pos<0) {    // pas de mise en cache data, vΘrifier existence
  1163.               /* note: no check with IS_DELAYED_EXT() enabled - postcheck by client please! */
  1164.               if (!IS_DELAYED_EXT(save) && fsize(fconv(save)) <= 0) {  // fichier existe pas ou est vide!
  1165.                 int found=0;
  1166.  
  1167.                 /* It is possible that the file has been moved due to changes in build structure */
  1168.                 {
  1169.                   char BIGSTK previous_save[HTS_URLMAXSIZE*2];
  1170.                   previous_save[0] = '\0';
  1171.                   back[p].r = cache_readex(opt, cache, adr, fil, NULL, back[p].location_buffer, previous_save, 0);
  1172.                   if (previous_save[0] != '\0' && fexist(fconv(previous_save))) {
  1173.                     rename(fconv(previous_save), fconv(save));
  1174.                     if (fexist(fconv(save))) {
  1175.                       found = 1;
  1176.                       if ((opt->debug>1) && (opt->log!=NULL)) {
  1177.                         fspc(opt->log,"debug"); fprintf(opt->log,"File '%s' has been renamed since last mirror to '%s' ; applying changes"LF, previous_save, save); test_flush;
  1178.                       }
  1179.                     } else {
  1180.                       if ((opt->debug>0) && (opt->log!=NULL)) {
  1181.                         fspc(opt->log,"error"); fprintf(opt->log,"Could not rename '%s' to '%s' ; will have to retransfer it"LF, previous_save, save); test_flush;
  1182.                       }
  1183.                     }
  1184.                   }
  1185.                 }
  1186.                 
  1187.                 if (!found) {
  1188. #if HTS_FAST_CACHE
  1189.                   hash_pos_return=0;
  1190. #else
  1191.                   a=NULL;    
  1192. #endif
  1193.                   // dΘvalider car non prΘsent sur disque dans structure originale!!!
  1194.                   // sinon, le fichier est ok α priori, mais on renverra un if-modified-since pour
  1195.                   // en Ωtre s√r
  1196.                   if (opt->norecatch) {              // tester norecatch
  1197.                     if (!fexist(fconv(save))) {  // fichier existe pas mais dΘclarΘ: on l'a effacΘ
  1198.                       FILE* fp=fopen(fconv(save),"wb");
  1199.                       if (fp) fclose(fp);
  1200.                       if (opt->log!=NULL) {
  1201.                         fspc(opt->log,"warning"); fprintf(opt->log,"File must have been erased by user, ignoring: %s%s"LF,back[p].url_adr,back[p].url_fil); test_flush;
  1202.                       }
  1203.                     }
  1204.                   }
  1205.                 }
  1206.               }  // fsize() <= 0
  1207.             }
  1208.           }
  1209.         }
  1210.         //
  1211.       } else
  1212. #if HTS_FAST_CACHE
  1213.         hash_pos_return=0;
  1214. #else
  1215.         a=NULL;
  1216. #endif
  1217.  
  1218.       // Existe pas en cache, ou bien pas de cache prΘsent
  1219. #if HTS_FAST_CACHE
  1220.       if (hash_pos_return) {  // OK existe en cache (et donnΘes aussi)!
  1221. #else
  1222.       if (a!=NULL) {  // OK existe en cache (et donnΘes aussi)!
  1223. #endif
  1224.         if (cache->type==1) {   // cache prioritaire (pas de test if-modified..)
  1225.                                // dans ce cas on peut Θgalement lire des rΘponses cachΘes comme 404,302...
  1226.           // lire dans le cache
  1227.           if (!test)
  1228.             back[p].r = cache_read(opt,cache,adr,fil,save, back[p].location_buffer);
  1229.           else
  1230.             back[p].r = cache_read(opt,cache,adr,fil,NULL, back[p].location_buffer);  // charger en tΩte uniquement du cache
  1231.  
  1232.           /* ensure correct location buffer set */
  1233.           back[p].r.location=back[p].location_buffer;
  1234.  
  1235.           /* Interdiction taille par le wizard? --> dΘtruire */
  1236.           if (back[p].r.statuscode != -1) {  // pas d'erreur de lecture
  1237.             if (!back_checksize(opt,&back[p],0)) {
  1238.               back[p].status=0;  // FINI
  1239.               back_set_finished(sback, p);
  1240.               back[p].r.statuscode=STATUSCODE_TOO_BIG;
  1241.               if (!back[p].testmode)
  1242.                 strcpybuff(back[p].r.msg,"Cached file skipped (too big)");
  1243.               else
  1244.                 strcpybuff(back[p].r.msg,"Test: Cached file skipped  (too big)");
  1245.               return 0;
  1246.             }
  1247.           }
  1248.  
  1249.           if (back[p].r.statuscode != -1 || IS_DELAYED_EXT(save)) {  // pas d'erreur de lecture ou test retardΘ
  1250.             if ((opt->debug>0) && (opt->log!=NULL)) {
  1251.               if (!test) {
  1252.                 fspc(opt->log,"debug"); fprintf(opt->log,"File immediately loaded from cache: %s%s"LF,back[p].url_adr,back[p].url_fil); test_flush;
  1253.               } else {
  1254.                 fspc(opt->log,"debug"); fprintf(opt->log,"File immediately tested from cache: %s%s"LF,back[p].url_adr,back[p].url_fil); test_flush;
  1255.               }
  1256.             }
  1257.             back[p].r.notmodified=1;    // fichier non modifiΘ
  1258.             back[p].status=0;  // OK prΩt
  1259.             //file_notify(back[p].url_adr, back[p].url_fil, back[p].url_sav, 0, 0, back[p].r.notmodified);        // not modified
  1260.             back_set_finished(sback, p);
  1261.  
  1262.             // finalize transfer
  1263.             if (!test) {
  1264.               if (back[p].r.statuscode>0) {
  1265.                 back_finalize(opt,cache,sback,p);
  1266.               }
  1267.             }
  1268.  
  1269.             return 0;
  1270.           } else {  // erreur
  1271.             // effacer r
  1272.             memset(&(back[p].r), 0, sizeof(htsblk)); back[p].r.soc=INVALID_SOCKET; back[p].r.location=back[p].location_buffer;
  1273.             // et continuer (chercher le fichier)
  1274.           }
  1275.           
  1276.         } else if (cache->type==2) {    // si en cache, demander de tester If-Modified-Since
  1277.           htsblk r;
  1278.           cache_header(opt,cache,adr,fil,&r);
  1279.  
  1280.           /* Interdiction taille par le wizard? */
  1281.           {
  1282.             LLint save_totalsize=back[p].r.totalsize;
  1283.             back[p].r.totalsize=r.totalsize;
  1284.             if (!back_checksize(opt,&back[p],1)) {
  1285.               r.statuscode = STATUSCODE_INVALID;
  1286.               //
  1287.               back[p].status=0;  // FINI
  1288.               back_set_finished(sback, p);
  1289.               back[p].r.statuscode=STATUSCODE_TOO_BIG;
  1290.               deletehttp(&back[p].r); back[p].r.soc=INVALID_SOCKET;
  1291.               if (!back[p].testmode)
  1292.                 strcpybuff(back[p].r.msg,"File too big");
  1293.               else
  1294.                 strcpybuff(back[p].r.msg,"Test: File too big");
  1295.               return 0;
  1296.             }
  1297.             back[p].r.totalsize=save_totalsize;
  1298.           }
  1299.           
  1300.           if (r.statuscode != -1) {
  1301.             if (r.statuscode==200) {     // uniquement des 200 (OK)
  1302.               if (strnotempty(r.etag)) {  // ETag (RFC2616)
  1303.                 /*
  1304.                 - If both an entity tag and a Last-Modified value have been
  1305.                 provided by the origin server, SHOULD use both validators in
  1306.                 cache-conditional requests. This allows both HTTP/1.0 and
  1307.                 HTTP/1.1 caches to respond appropriately.
  1308.                 */
  1309.                 if (strnotempty(r.lastmodified))
  1310.                   sprintf(back[p].send_too,"If-None-Match: %s\r\nIf-Modified-Since: %s\r\n",r.etag,r.lastmodified);
  1311.                 else
  1312.                   sprintf(back[p].send_too,"If-None-Match: %s\r\n",r.etag);
  1313.               }
  1314.               else if (strnotempty(r.lastmodified))
  1315.                 sprintf(back[p].send_too,"If-Modified-Since: %s\r\n",r.lastmodified);
  1316.               else if (strnotempty(cache->lastmodified))
  1317.                 sprintf(back[p].send_too,"If-Modified-Since: %s\r\n",cache->lastmodified);
  1318.               
  1319.               /* this is an update of a file */
  1320.               if (strnotempty(back[p].send_too))
  1321.                 back[p].is_update=1;
  1322.               back[p].r.req.nocompression=1;   /* Do not compress when updating! */
  1323.               
  1324.             }
  1325.           }
  1326. #if DEBUGCA
  1327.           printf("..is modified test %s\n",back[p].send_too);
  1328. #endif
  1329.         } 
  1330.         // Okay, pas trouvΘ dans le cache
  1331.         // Et si le fichier existe sur disque?
  1332.         // Pas dans le cache: fichier n'a pas ΘtΘ transfΘrΘ du tout, donc pas sur disque?
  1333.       } else {
  1334.         if (fexist(save)) {    // fichier existe? aghl!
  1335.           LLint sz=fsize(save);
  1336.           // Bon, lα il est possible que le fichier ait ΘtΘ partiellement transfΘrΘ
  1337.           // (s'il l'avait ΘtΘ en totalitΘ il aurait ΘtΘ inscrit dans le cache ET existerait sur disque)
  1338.           // PAS de If-Modified-Since, on a pas connaissance des donnΘes α la date du cache
  1339.           // On demande juste les donnΘes restantes si le date est valide (206), tout sinon (200)
  1340.           if ((ishtml(save) != 1) && (ishtml(back[p].url_fil)!=1)) {   // NON HTML (liens changΘs!!)
  1341.             if (sz>0) {    // Fichier non vide? (question bΩte, sinon on transfert tout!)
  1342.               char lastmodified[256];
  1343.               get_filetime_rfc822(save, lastmodified);
  1344.               if (strnotempty(lastmodified)) {     /* pas de If-.. possible */
  1345. #if DEBUGCA
  1346.                 printf("..if unmodified since %s size "LLintP"\n", lastmodified, (LLint)sz);
  1347. #endif
  1348.                 if ((opt->debug>1) && (opt->log!=NULL)) {
  1349.                   fspc(opt->log,"debug"); fprintf(opt->log,"File partially present ("LLintP" bytes): %s%s"LF,(LLint)sz,back[p].url_adr,back[p].url_fil); test_flush;
  1350.                 }
  1351.                 
  1352.                 /* impossible - don't have etag or date
  1353.                 if (strnotempty(back[p].r.etag)) {  // ETag (RFC2616)
  1354.                 sprintf(back[p].send_too,"If-None-Match: %s\r\n",back[p].r.etag);
  1355.                 back[p].http11=1;    // En tΩte 1.1
  1356.                 } else if (strnotempty(back[p].r.lastmodified)) {
  1357.                 sprintf(back[p].send_too,"If-Unmodified-Since: %s\r\n",back[p].r.lastmodified);
  1358.                 back[p].http11=1;    // En tΩte 1.1
  1359.                 } else 
  1360.                 */
  1361.                 if (strlen(lastmodified)) {
  1362.                   sprintf(back[p].send_too,
  1363.                     "If-Unmodified-Since: %s\r\nRange: bytes="LLintP"-\r\n"
  1364.                     , lastmodified, (LLint)sz);
  1365.                   back[p].http11=1;    // En tΩte 1.1
  1366.                   back[p].range_req_size=sz;
  1367.                   back[p].r.req.range_used=1;
  1368.                   back[p].r.req.nocompression=1;
  1369.                 } else {
  1370.                   if ((opt->debug>0) && (opt->log!=NULL)) {
  1371.                     fspc(opt->log,"warning"); fprintf(opt->log,"Could not find timestamp for partially present file, restarting (lost "LLintP" bytes): %s%s"LF,(LLint)sz,back[p].url_adr,back[p].url_fil); test_flush;
  1372.                   }
  1373.                 }
  1374.                 
  1375.               } else { 
  1376.                 if ((opt->debug>0) && (opt->errlog!=NULL)) {
  1377.                   fspc(opt->errlog,"warning");
  1378.                   /*
  1379.                   if (opt->http10)
  1380.                   fprintf(opt->errlog,"File partially present (%d bytes) retransfered due to HTTP/1.0 settings: %s%s"LF,sz,back[p].url_adr,back[p].url_fil);
  1381.                   else
  1382.                   */
  1383.                   fprintf(opt->errlog,"File partially present ("LLintP" bytes) retransfered due to lack of cache: %s%s"LF,(LLint)sz,back[p].url_adr,back[p].url_fil); 
  1384.                   test_flush;
  1385.                 }
  1386.                 /* Sinon requΩte normale... */
  1387.                 back[p].http11=0;
  1388.               }
  1389.             } else if (opt->norecatch) {              // tester norecatch
  1390.               filenote(save,NULL);       // ne pas purger tout de mΩme
  1391.               file_notify(back[p].url_adr, back[p].url_fil, back[p].url_sav, 0, 0, back[p].r.notmodified);
  1392.               back[p].status=0;  // OK prΩt
  1393.               back_set_finished(sback, p);
  1394.               back[p].r.statuscode=STATUSCODE_INVALID;  // erreur
  1395.               strcpybuff(back[p].r.msg,"Null-size file not recaught");
  1396.               return 0;
  1397.             }
  1398.           } else {
  1399.             if ((opt->debug>0) && (opt->errlog!=NULL)) {
  1400.               fspc(opt->errlog,"warning");
  1401.               fprintf(opt->errlog,"HTML file ("LLintP" bytes) retransfered due to lack of cache: %s%s"LF,(LLint)sz,back[p].url_adr,back[p].url_fil); 
  1402.               test_flush;
  1403.             }
  1404.             /* Sinon requΩte normale... */
  1405.             back[p].http11=0;
  1406.           }
  1407.         }
  1408.       }
  1409.     }
  1410.  
  1411.  
  1412.     {
  1413.       ///htsblk r;   non directement dans la structure-rΘponse!
  1414.       T_SOC soc;
  1415.       
  1416.       // ouvrir liaison, envoyer requΦte
  1417.       // ne pas traiter ou recevoir l'en tΩte immΘdiatement
  1418.       memset(&(back[p].r), 0, sizeof(htsblk)); back[p].r.soc=INVALID_SOCKET; back[p].r.location=back[p].location_buffer;
  1419.       // recopier proxy
  1420.       memcpy(&(back[p].r.req.proxy), &opt->proxy, sizeof(opt->proxy));
  1421.       // et user-agent
  1422.       strcpybuff(back[p].r.req.user_agent,opt->user_agent);
  1423.       strcpybuff(back[p].r.req.referer,opt->referer);
  1424.       strcpybuff(back[p].r.req.from,opt->from);
  1425.       strcpybuff(back[p].r.req.lang_iso,opt->lang_iso);
  1426.       back[p].r.req.user_agent_send=opt->user_agent_send;
  1427.       // et http11
  1428.       back[p].r.req.http11=back[p].http11;
  1429.       back[p].r.req.nocompression=opt->nocompression;
  1430.       back[p].r.req.nokeepalive=opt->nokeepalive;
  1431.  
  1432.       // mode ftp, court-circuit!
  1433.       if (strfield(back[p].url_adr,"ftp://")) {
  1434.         if (back[p].testmode) {
  1435.           if ((opt->debug>1) && (opt->errlog!=NULL)) {
  1436.             fspc(opt->errlog,"debug"); fprintf(opt->errlog,"error: forbidden test with ftp link for back_add"LF);
  1437.           }
  1438.           return -1;    // erreur pas de test permis
  1439.         }
  1440.         if (!(back[p].r.req.proxy.active && opt->ftp_proxy)) { // connexion directe, gΘrΘe en thread
  1441.           back[p].status=1000;   // connexion ftp
  1442. #if USE_BEGINTHREAD
  1443.           launch_ftp(&(back[p]));
  1444. #else
  1445.           {
  1446.             char nid[32];
  1447.             sprintf(nid,"htsftp%d-in_progress.lock",p);
  1448.             strcpybuff(back[p].location_buffer,fconcat(opt->path_log,nid));
  1449.           }
  1450.           launch_ftp(&(back[p]),back[p].location_buffer,opt->exec);
  1451. #endif
  1452.           return 0;
  1453.         }
  1454.       }
  1455. #if HTS_USEMMS
  1456.       else if (strfield(back[p].url_adr,"mms://")) {
  1457.                 MMSDownloadStruct str;
  1458.         if (back[p].testmode) {
  1459.           if ((opt->debug>1) && (opt->errlog!=NULL)) {
  1460.             fspc(opt->errlog,"debug"); fprintf(opt->errlog,"error: forbidden test with mms link for back_add"LF);
  1461.           }
  1462.           return -1;    // erreur pas de test permis
  1463.         }
  1464.         if (back[p].r.req.proxy.active) {
  1465.           fspc(opt->errlog,"warning"); fprintf(opt->errlog,"warning: direct connection for mms links (proxy settings ignored)"LF);
  1466.                 }
  1467.                 back[p].status=1000;   // connexion externe
  1468.                 str.pBack = &back[p];
  1469.                 str.pOpt = opt;
  1470.                 launch_mms(&str);
  1471.                 return 0;
  1472.       }
  1473. #endif
  1474. #if HTS_USEOPENSSL
  1475.       else if (SSL_is_available && strfield(back[p].url_adr,"https://")) {        // let's rock
  1476.         back[p].r.ssl = 1;
  1477.         // back[p].r.ssl_soc = NULL;
  1478.         back[p].r.ssl_con = NULL;
  1479.       }
  1480. #endif
  1481.       
  1482.       if (!back_trylive(opt, cache, sback, p)) {
  1483. #if HTS_XGETHOST
  1484. #if HDEBUG
  1485.         printf("back_solve..\n");
  1486. #endif
  1487.         back[p].status=101;    // tentative de rΘsolution du nom de host
  1488.         soc=INVALID_SOCKET;    // pas encore ouverte
  1489.         back_solve(&back[p]);  // prΘparer
  1490.         if (host_wait(&back[p])) {  // prΩt, par ex fichier ou dispo dans dns
  1491. #if HDEBUG
  1492.           printf("ok, dns cache ready..\n");
  1493. #endif
  1494.           soc=http_xfopen(0,0,0,back[p].send_too,adr,fil,&(back[p].r));
  1495.           if (soc==INVALID_SOCKET) {
  1496.             back[p].status=0;  // fini, erreur
  1497.             back_set_finished(sback, p);
  1498.           }
  1499.         }
  1500.         //
  1501. #else
  1502.         //
  1503. #if CNXDEBUG
  1504.         printf("XFopen..\n");
  1505. #endif
  1506.         
  1507.         if (strnotempty(back[p].send_too))    // envoyer un if-modified-since
  1508. #if HTS_XCONN
  1509.           soc=http_xfopen(0,0,0,back[p].send_too,adr,fil,&(back[p].r));
  1510. #else
  1511.         soc=http_xfopen(0,0,1,back[p].send_too,adr,fil,&(back[p].r));
  1512. #endif
  1513.         else
  1514. #if HTS_XCONN
  1515.           soc=http_xfopen(test,0,0,NULL,adr,fil,&(back[p].r));
  1516. #else
  1517.         soc=http_xfopen(test,0,1,NULL,adr,fil,&(back[p].r));
  1518. #endif
  1519. #endif
  1520.       } else {
  1521.         soc = back[p].r.soc;
  1522.  
  1523.         if ((opt->debug>1) && (opt->log!=NULL)) {
  1524.           fspc(opt->log,"debug"); fprintf(opt->log,"(Keep-Alive): successfully linked #%d (for %s%s)"LF, 
  1525.             back[p].r.debugid,
  1526.             back[p].url_adr, back[p].url_fil); test_flush;
  1527.         }
  1528.       }
  1529.       
  1530.       if (opt->timeout>0) {    // gestion du opt->timeout
  1531.         back[p].timeout=opt->timeout;
  1532.         back[p].timeout_refresh=time_local();
  1533.       } else {
  1534.         back[p].timeout=-1;    // pas de gestion (default)
  1535.       }
  1536.       
  1537.       if (opt->rateout>0) {    // gestion d'un taux minimum de transfert tolΘrΘ
  1538.         back[p].rateout=opt->rateout;
  1539.         back[p].rateout_time=time_local();
  1540.       } else {
  1541.         back[p].rateout=-1;    // pas de gestion (default)
  1542.       }
  1543.  
  1544.       // Note: on charge les code-page erreurs (erreur 404, etc) dans le cas o∙ cela est
  1545.       // rattrapable (exemple: 301,302 moved xxx -> refresh sur la page!)
  1546.       //if ((back[p].statuscode!=200) || (soc<0)) { // ERREUR HTTP/autre
  1547.  
  1548. #if CNXDEBUG
  1549. printf("Xfopen ok, poll..\n");
  1550. #endif
  1551.  
  1552. #if HTS_XGETHOST
  1553.     if (soc!=INVALID_SOCKET)
  1554.       if (back[p].status==101) {  // pas d'erreur
  1555.         if (!back[p].r.is_file)
  1556.           back[p].status=100;   // connexion en cours
  1557.         else
  1558.           back[p].status=1;     // fichier
  1559.       }
  1560.  
  1561. #else
  1562.       if (soc==INVALID_SOCKET) { // erreur socket
  1563.         back[p].status=0;    // FINI
  1564.         back_set_finished(sback, p);
  1565.         //if (back[p].soc!=INVALID_SOCKET) deletehttp(back[p].soc);
  1566.         back[p].r.soc=INVALID_SOCKET;
  1567.       } else {
  1568.         if (!back[p].r.is_file)
  1569. #if HTS_XCONN
  1570.           back[p].status=100;   // connexion en cours
  1571. #else
  1572.           back[p].status=99;    // chargement en tΩte en cours
  1573. #endif
  1574.         else
  1575.           back[p].status=1;     // chargement fichier
  1576. #if BDEBUG==1
  1577.         printf("..loading header\n");
  1578. #endif
  1579.       }
  1580. #endif
  1581.       
  1582.     }
  1583.  
  1584.  
  1585.     // note: si il y a erreur (404,etc) status=2 (terminΘ/Θchec) mais
  1586.     // le lien est considΘrΘ comme traitΘ
  1587.     //if (back[p].soc<0)  // erreur
  1588.     //  return -1;
  1589.  
  1590.     return 0;
  1591.   } else {
  1592.     if ((opt->debug>1) && (opt->errlog!=NULL)) {
  1593.       fspc(opt->errlog,"debug"); fprintf(opt->errlog,"error: no space left in stack for back_add"LF);
  1594.     }
  1595.     return -1;    // plus de place
  1596.   }
  1597. }
  1598.  
  1599.  
  1600.  
  1601. #if HTS_XGETHOST
  1602. #if USE_BEGINTHREAD
  1603. // lancement multithread du robot
  1604. PTHREAD_TYPE PTHREAD_TYPE_FNC Hostlookup(void* iadr_p) {
  1605.   char iadr[256];
  1606.   t_dnscache* cache=_hts_cache();  // adresse du cache
  1607.   t_hostent* hp;
  1608.   int error_found=0;
  1609.  
  1610.   // recopier (aprΦs id:pass)
  1611. #if DEBUGDNS 
  1612.   printf("resolv in background: %s\n",jump_identification(iadr_p));
  1613. #endif
  1614.   strcpybuff(iadr,jump_identification(iadr_p));
  1615.   // couper Θventuel :
  1616.   {
  1617.     char *a;
  1618.     if ( (a=jump_toport(iadr)) )
  1619.       *a='\0';          // get rid of it
  1620.   }
  1621.   freet(iadr_p);
  1622.  
  1623.   // attendre que le cache dns soit prΩt
  1624.   while(_hts_lockdns(-1));  // attendre libΘration
  1625.   _hts_lockdns(1);          // locker
  1626.   while(cache->n) {
  1627.     if (strcmp(cache->iadr,iadr)==0) {
  1628.       error_found=1;
  1629.     }
  1630.     cache=cache->n;    // calculer queue
  1631.   }
  1632.   if (strcmp(cache->iadr,iadr)==0) {
  1633.     error_found=1;
  1634.   }
  1635.  
  1636.   if (!error_found) {
  1637.     // en gros copie de hts_gethostbyname sans le return
  1638.     cache->n=(t_dnscache*) calloct(1,sizeof(t_dnscache));
  1639.     if (cache->n!=NULL) {
  1640.       t_fullhostent fullhostent_buffer;
  1641.       strcpybuff(cache->n->iadr,iadr);
  1642.       cache->n->host_length=0;        /* pour le moment rien */
  1643.       cache->n->n=NULL;
  1644.       _hts_lockdns(0);          // dΘlocker
  1645.       
  1646.       /* resolve */
  1647. #if DEBUGDNS 
  1648.       printf("gethostbyname() in progress for %s\n",iadr);
  1649. #endif
  1650.       cache->n->host_length=-1;
  1651.       memset(cache->n->host_addr, 0, sizeof(cache->n->host_addr));
  1652.       hp=vxgethostbyname(iadr, &fullhostent_buffer);
  1653.       if (hp!=NULL) {
  1654.         memcpy(cache->n->host_addr, hp->h_addr, hp->h_length);
  1655.         cache->n->host_length = hp->h_length;
  1656.       }
  1657.     } else 
  1658.     _hts_lockdns(0);          // dΘlocker
  1659.   } else {
  1660. #if DEBUGDNS 
  1661.     printf("aborting resolv for %s (found)\n",iadr);
  1662. #endif
  1663.     _hts_lockdns(0);          // dΘlocker
  1664.   }
  1665.   // fin de copie de hts_gethostbyname
  1666.  
  1667. #if DEBUGDNS 
  1668.   printf("quitting resolv for %s (result: %d)\n",iadr,(cache->n!=NULL)?cache->n->host_length:(-999));
  1669. #endif
  1670.  
  1671.   return PTHREAD_RETURN;     /* _endthread implied  */
  1672. }
  1673. #endif
  1674.  
  1675. // attendre que le host (ou celui du proxy) ait ΘtΘ rΘsolu
  1676. // si c'est un fichier, la rΘsolution est immΘdiate
  1677. // idem pour ftp://
  1678. void back_solve(lien_back* back) {
  1679.   if ((!strfield(back->url_adr,"file://")) 
  1680.         && ! strfield(back->url_adr,"ftp://")
  1681. #if HTS_USEMMS
  1682.         && ! strfield(back->url_adr,"mms://")
  1683. #endif
  1684.         ) {
  1685.   //## if (back->url_adr[0]!=lOCAL_CHAR) {  // qq chose α prΘparer
  1686.     char* a;
  1687.     if (!(back->r.req.proxy.active))
  1688.       a=back->url_adr;
  1689.     else
  1690.       a=back->r.req.proxy.name;
  1691.     a = jump_protocol(a);
  1692.     if (!hts_dnstest(a)) {   // non encore testΘ!..
  1693.       // inscire en thread
  1694. #if HTS_WIN
  1695.       // Windows
  1696. #if USE_BEGINTHREAD
  1697.       {
  1698.         char* p = calloct(strlen(a)+2,1);
  1699.         if (p) {
  1700.           strcpybuff(p,a);
  1701.           (void)hts_newthread( Hostlookup , 0, p );
  1702.         }
  1703.       }
  1704. #else
  1705.       /*t_hostent* h=*/
  1706.       /*hts_gethostbyname(a);*/  // calcul
  1707. #endif
  1708. #else
  1709. #if USE_BEGINTHREAD
  1710.         char* p = calloct(strlen(a)+2,1);
  1711.         if (p) {
  1712.           strcpybuff(p,a);
  1713.           (void)hts_newthread( Hostlookup , 0, p );
  1714.         }
  1715. #else
  1716.       // Sous Unix, le gethostbyname() est bloquant..
  1717.       /*t_hostent* h=*/
  1718.       /*hts_gethostbyname(a);*/  // calcul
  1719. #endif
  1720. #endif
  1721.     }
  1722.   }
  1723. }
  1724.  
  1725. // dΘtermine si le host a pu Ωtre rΘsolu
  1726. int host_wait(lien_back* back) {
  1727.   if ((!strfield(back->url_adr,"file://")) 
  1728.         && (!strfield(back->url_adr,"ftp://"))
  1729. #if HTS_USEMMS
  1730.         && (!strfield(back->url_adr,"mms://"))
  1731. #endif
  1732.         ) {
  1733.   //## if (back->url_adr[0]!=lOCAL_CHAR) {
  1734.     if (!(back->r.req.proxy.active)) {
  1735.       return (hts_dnstest(back->url_adr));
  1736.     } else {
  1737.       return (hts_dnstest(back->r.req.proxy.name));      
  1738.     }
  1739.   } else return 1;    // prΩt, fichier local
  1740. }
  1741. #endif
  1742.  
  1743.  
  1744. // Θlimine les fichiers non html en backing (anticipation)
  1745. // cleanup non-html files in backing to save backing space
  1746. // and allow faster "save in cache" operation
  1747. // also cleanup keep-alive sockets and ensure that not too many sockets are being opened
  1748. void back_clean(httrackp* opt,cache_back* cache,struct_back* sback) {
  1749.   lien_back* const back = sback->lnk;
  1750.   const int back_max = sback->count;
  1751. #if HTS_ANALYSTE
  1752.   int oneMore = ( (_hts_in_html_parsing == 2 && opt->maxsoc >= 2) || (_hts_in_html_parsing == 1 && opt->maxsoc >= 4) ) ? 1 : 0;  // testing links
  1753. #endif
  1754.   int i;
  1755.   for(i=0;i<back_max;i++) {
  1756.     if (back[i].status == 0) {                                   // ready
  1757.       /* Check autoclean */
  1758.       if (!back[i].testmode) {                                   // not test mode
  1759.         if (strnotempty(back[i].url_sav)) {                      // filename exists
  1760.           if (back[i].r.statuscode==200) {                       // HTTP "OK"
  1761.             if (back[i].r.size>0) {                              // size>0
  1762.               if (back[i].r.is_write                             // not in memory (on disk, ready)
  1763.                 && !is_hypertext_mime(back[i].r.contenttype, back[i].url_fil)        // not HTML/hypertext
  1764.                 && !may_be_hypertext_mime(back[i].r.contenttype, back[i].url_fil)    // may NOT be parseable mime type
  1765.                 ) 
  1766.               {
  1767.                 if (back[i].pass2_ptr) {
  1768.                                     (void) back_flush_output(opt, cache, sback, i);        // flush output buffers
  1769.                   usercommand(opt, 0, NULL, back[i].url_sav, back[i].url_adr, back[i].url_fil);
  1770.                   *back[i].pass2_ptr=-1;  // Done!
  1771.                   HTS_STAT.stat_background++;
  1772.                   if ((opt->debug>0) && (opt->log!=NULL)) {
  1773.                     fspc(opt->log,"info"); fprintf(opt->log,"File successfully written in background: %s"LF,back[i].url_sav); test_flush;
  1774.                   }
  1775.                   back_maydelete(opt,cache,sback,i);    // May delete backing entry
  1776.                 }
  1777.               } else {
  1778.                 if (!back[i].finalized) {
  1779.                   if (1) {
  1780.                     /* Ensure deleted or recycled socket */
  1781.                     /* BUT DO NOT YET WIPE back[i].r.adr */
  1782.                     if ( (opt->debug>1) && (opt->log!=NULL) ) {
  1783.                       fspc(opt->log,"debug"); fprintf(opt->log,"file %s%s validated (cached, left in memory)"LF,back[i].url_adr,back[i].url_fil); test_flush;
  1784.                     }
  1785.                     back_maydeletehttp(opt, cache, sback, i);
  1786.                   } else {
  1787.                     /*
  1788.                     NOT YET HANDLED CORRECTLY (READ IN NEW CACHE TO DO)
  1789.                     */
  1790.                     /* Lock the entry but do not keep the html data in memory (in cache) */
  1791.                     if (opt->cache) {
  1792.                       htsblk r;
  1793.  
  1794.                       /* Ensure deleted or recycled socket */
  1795.                       back_maydeletehttp(opt, cache, sback, i);
  1796.                       assertf(back[i].r.soc == INVALID_SOCKET);
  1797.  
  1798.                       /* Check header */
  1799.                       cache_header(opt,cache,back[i].url_adr,back[i].url_fil,&r);
  1800.                       if (r.statuscode == 200) {
  1801.                         if (back[i].r.soc == INVALID_SOCKET) {
  1802.                           /* Delete buffer and sockets */
  1803.                           deleteaddr(&back[i].r);
  1804.                           deletehttp(&back[i].r);
  1805.                           if ( (opt->debug>1) && (opt->log!=NULL) ) {
  1806.                             fspc(opt->log,"debug"); fprintf(opt->log,"file %s%s temporarily left in cache to spare memory"LF,back[i].url_adr,back[i].url_fil); test_flush;
  1807.                           }
  1808.                         }
  1809.                       } else {
  1810.                         if ((opt->debug>0) && (opt->log!=NULL)) {
  1811.                           fspc(opt->log,"warning"); fprintf(opt->log,"Unexpected html cache lookup error during back clean"LF); test_flush;
  1812.                         }            
  1813.                       }
  1814.                       // xxc xxc
  1815.                     }
  1816.                   }
  1817.                 }
  1818.               }
  1819.             }
  1820.           }
  1821.         }
  1822.       }
  1823.     } else if (back[i].status == -103) {                         // waiting (keep-alive)
  1824.       if (
  1825.         ! back[i].r.keep_alive
  1826.         || back[i].r.soc == INVALID_SOCKET
  1827.         || back[i].r.keep_alive_max < 1
  1828.         || time_local() >= back[i].ka_time_start + back[i].r.keep_alive_t
  1829.         ) {
  1830.         if ((opt->debug>0) && (opt->log!=NULL)) {
  1831.             fspc(opt->log,"debug"); fprintf(opt->log,"(Keep-Alive): live socket closed #%d (%s)"LF, 
  1832.             back[i].r.debugid,
  1833.             back[i].url_adr);
  1834.             test_flush;
  1835.         }
  1836.         back_delete(opt,cache,sback, i);    // delete backing entry
  1837.       }
  1838.     }
  1839.   }
  1840.   /* switch connections to live ones */
  1841.   for(i=0;i<back_max;i++) {
  1842.     if (back[i].status == 0) {                                   // ready
  1843.       if (back[i].r.soc != INVALID_SOCKET) {
  1844.         back_maydeletehttp(opt,cache,sback, i);
  1845.       }
  1846.     }
  1847.   }
  1848.   /* delete sockets if too many keep-alive'd sockets in background */
  1849.   if (opt->maxsoc > 0) {
  1850.     int max = opt->maxsoc + oneMore;
  1851.     int curr = back_nsoc_overall(sback);
  1852.     if (curr > max) {
  1853.       if ((opt->debug>1) && (opt->log!=NULL)) {
  1854.         fspc(opt->log,"debug"); fprintf(opt->log,"(Keep-Alive): deleting #%d sockets"LF, 
  1855.           curr - max); test_flush;
  1856.       }
  1857.     }
  1858.     for(i = 0 ; i < back_max && curr > max ; i++) {
  1859.       if (back[i].status == -103) {
  1860.         back_delete(opt,cache,sback, i);    // delete backing entry
  1861.         curr--;
  1862.       }
  1863.     }
  1864.   }
  1865.   /* transfer ready slots to the storage hashtable */
  1866.   {
  1867.     int nxfr = back_cleanup_background(opt,cache,sback);
  1868.     if (nxfr > 0 && (opt->debug>0) && (opt->log!=NULL)) {
  1869.       fspc(opt->log,"debug"); fprintf(opt->log,"(htsback): %d slots ready moved to background"LF, nxfr);
  1870.       test_flush;
  1871.     }
  1872.   }
  1873. }
  1874.  
  1875.  
  1876. // attente (gestion des buffers des sockets)
  1877. void back_wait(struct_back* sback,httrackp* opt,cache_back* cache,TStamp stat_timestart) {
  1878.   lien_back* const back = sback->lnk;
  1879.   const int back_max = sback->count;
  1880.   unsigned int i_mod;
  1881.   T_SOC nfds=INVALID_SOCKET;
  1882.   fd_set fds,fds_c,fds_e;     // fds pour lecture, connect (write), et erreur
  1883.   int nsockets;     // nbre sockets
  1884.   LLint max_read_bytes;  // max bytes read per sockets
  1885.   struct timeval tv;
  1886.   int do_wait=0;
  1887.   int gestion_timeout=0;
  1888.   int busy_recv=0;     // pas de donnΘes pour le moment   
  1889.   int busy_state=0;    // pas de connexions
  1890.   int max_loop;  // nombre de boucles max α parcourir..
  1891. #if HTS_ANALYSTE
  1892.   int max_loop_chk=0;
  1893. #endif
  1894.   unsigned int mod_random = (unsigned int) ( time_local() + HTS_STAT.HTS_TOTAL_RECV );
  1895.  
  1896.   // max. number of loops
  1897.   max_loop=8;
  1898.  
  1899. #if 1
  1900.   // Cleanup the stack to save space!
  1901.   back_clean(opt,cache,sback);
  1902. #endif
  1903.  
  1904.   // recevoir tant qu'il y a des donnΘes (avec un maximum de max_loop boucles)
  1905.   do_wait=0;
  1906.   gestion_timeout=0;
  1907.   do {
  1908.     int max_c;
  1909.     busy_state=busy_recv=0;
  1910.  
  1911. #if 0
  1912.     check_rate(stat_timestart,opt->maxrate);    // vΘrifier taux de transfert
  1913. #endif
  1914.     // inscrire les sockets actuelles, et rechercher l'ID la plus ΘlevΘe
  1915.     FD_ZERO(&fds);
  1916.     FD_ZERO(&fds_c);
  1917.     FD_ZERO(&fds_e);
  1918.     nsockets=0;
  1919.     max_read_bytes=TAILLE_BUFFER;     // maximum bytes that can be read
  1920.     nfds=INVALID_SOCKET;
  1921.  
  1922.     max_c=1;
  1923.     for(i_mod = 0 ; i_mod < (unsigned int) back_max ; i_mod++) {
  1924.     // for(i=0;i<back_max;i++) {
  1925.         unsigned int i = ( i_mod + mod_random ) % ( back_max );
  1926.  
  1927.       // en cas de gestion du connect prΘemptif
  1928. #if HTS_XCONN
  1929.       if (back[i].status==100) {      // connexion
  1930.         do_wait=1;
  1931.  
  1932.         // noter socket write
  1933.         FD_SET(back[i].r.soc,&fds_c);
  1934.         
  1935.         // noter socket erreur
  1936.         FD_SET(back[i].r.soc,&fds_e);
  1937.  
  1938.         // calculer max
  1939.         if (max_c) {
  1940.           max_c=0;
  1941.           nfds=back[i].r.soc;
  1942.         } else if (back[i].r.soc>nfds) {
  1943.           // ID socket la plus ΘlevΘe
  1944.           nfds=back[i].r.soc;
  1945.         }
  1946.         
  1947.       } else
  1948. #endif
  1949. #if HTS_XGETHOST
  1950.       if (back[i].status==101) {      // attente
  1951.         // rien α faire..
  1952.       } else
  1953. #endif
  1954.       // poll pour la lecture sur les sockets
  1955.       if ((back[i].status>0) && (back[i].status<100)) {  // en rΘception http
  1956.             
  1957. #if BDEBUG==1
  1958.         //printf("....socket in progress: %d\n",back[i].r.soc);
  1959. #endif
  1960.         // non local et non ftp
  1961.         if (!back[i].r.is_file) {
  1962.         //## if (back[i].url_adr[0]!=lOCAL_CHAR) {
  1963.           
  1964.           // vΘrification de sΘcuritΘ
  1965.           if (back[i].r.soc!=INVALID_SOCKET) {  // hey, you never know..
  1966.             do_wait=1;
  1967.             
  1968.             // noter socket read
  1969.             FD_SET(back[i].r.soc,&fds);
  1970.             
  1971.             // noter socket error
  1972.             FD_SET(back[i].r.soc,&fds_e);
  1973.             
  1974.             // incrΘmenter nombre de sockets
  1975.             nsockets++;
  1976.  
  1977.             // calculer max
  1978.             if (max_c) {
  1979.               max_c=0;
  1980.               nfds=back[i].r.soc;
  1981.             } else if (back[i].r.soc>nfds) {
  1982.               // ID socket la plus ΘlevΘe
  1983.               nfds=back[i].r.soc;
  1984.             }
  1985.           } else {
  1986.             back[i].r.statuscode=STATUSCODE_CONNERROR;
  1987.             if (back[i].status==100)
  1988.               strcpybuff(back[i].r.msg,"Connect Error");
  1989.             else
  1990.               strcpybuff(back[i].r.msg,"Receive Error");
  1991.             back[i].status=0;  // terminΘ
  1992.             back_set_finished(sback, i);
  1993.             if ((opt->debug>0) && (opt->log!=NULL)) {
  1994.               fspc(opt->log,"warning"); fprintf(opt->log,"Unexpected socket error during pre-loop"LF); test_flush;
  1995.             }            
  1996.           }
  1997. #if WIDE_DEBUG
  1998.           else {
  1999.             DEBUG_W("PANIC!!! Socket is invalid in a poll test!\n");
  2000.           }
  2001. #endif
  2002.           
  2003.         }
  2004.         
  2005.       }
  2006.     }    
  2007.     nfds++;
  2008.     
  2009.     if (do_wait) {  // attendre
  2010.       // temps d'attente max: 2.5 seconde
  2011.       tv.tv_sec=HTS_SOCK_SEC;
  2012.       tv.tv_usec=HTS_SOCK_MS;
  2013.       
  2014. #if BDEBUG==1
  2015.       printf("..select\n");
  2016. #endif
  2017.       
  2018.       // poller les sockets-attention au noyau sous Unix..
  2019. #if HTS_WIDE_DEBUG    
  2020.       DEBUG_W("select\n");
  2021. #endif
  2022.       select(nfds,&fds,&fds_c,&fds_e,&tv);
  2023. #if HTS_WIDE_DEBUG    
  2024.       DEBUG_W("select done\n");
  2025. #endif      
  2026.     }
  2027.     
  2028.     // maximum data which can be received for a socket, if limited
  2029.     if (nsockets) {
  2030.       if (opt->maxrate>0) {
  2031.         max_read_bytes = ( check_downloadable_bytes(opt->maxrate) / nsockets );
  2032.         if (max_read_bytes > TAILLE_BUFFER) {
  2033.           /* limit size */
  2034.           max_read_bytes = TAILLE_BUFFER;
  2035.         } else if (max_read_bytes < TAILLE_BUFFER) {
  2036.           /* a small pause */
  2037.           Sleep(10);
  2038.         }
  2039.       }
  2040.     }
  2041.     if (!max_read_bytes)
  2042.       busy_recv=0;
  2043.     
  2044.     // recevoir les donnΘes arrivΘes
  2045.     for(i_mod = 0 ; i_mod < (unsigned int) back_max ; i_mod++) {
  2046.     // for(i=0;i<back_max;i++) {
  2047.         unsigned int i = ( i_mod + mod_random ) % ( back_max );     
  2048.       if (back[i].status>0) {
  2049.         if (!back[i].r.is_file) {  // not file..
  2050.           if (back[i].r.soc!=INVALID_SOCKET) {  // hey, you never know..
  2051.             int err=FD_ISSET(back[i].r.soc,&fds_e);
  2052.             if (err) {
  2053.               if (back[i].r.soc!=INVALID_SOCKET) {
  2054. #if HTS_DEBUG_CLOSESOCK
  2055.                 DEBUG_W("back_wait: deletehttp\n");
  2056. #endif
  2057.                 deletehttp(&back[i].r);
  2058.               }
  2059.               back[i].r.soc=INVALID_SOCKET;
  2060.               back[i].r.statuscode=STATUSCODE_CONNERROR;
  2061.               if (back[i].status==100)
  2062.                 strcpybuff(back[i].r.msg,"Connect Error");
  2063.               else
  2064.                 strcpybuff(back[i].r.msg,"Receive Error");
  2065.               if (back[i].status == -103) {     /* Keep-alive socket */
  2066.                 back_delete(opt,cache,sback, i);
  2067.               } else {
  2068.                 back[i].status=0;  // terminΘ
  2069.                 back_set_finished(sback, i);
  2070.               }
  2071.             }
  2072.           }
  2073.         }
  2074.       }
  2075.       
  2076.       // ---- FLAG WRITE MIS A UN?: POUR LE CONNECT
  2077.       if (back[i].status==100) {   // attendre connect
  2078.         int dispo=0;
  2079.         // vΘrifier l'existance de timeout-check
  2080.         if (!gestion_timeout)
  2081.           if (back[i].timeout>0)
  2082.             gestion_timeout=1;
  2083.           
  2084.           // connectΘ?
  2085.           dispo=FD_ISSET(back[i].r.soc,&fds_c);
  2086.           if (dispo) {    // ok connected!!
  2087.             busy_state=1;
  2088.             
  2089. #if HTS_USEOPENSSL
  2090.             /* SSL mode */
  2091.             if (SSL_is_available && back[i].r.ssl) {
  2092.               // handshake not yet launched
  2093.               if (!back[i].r.ssl_con) {
  2094.                 SSL_CTX_set_options(openssl_ctx, SSL_OP_ALL);
  2095.                 // new session
  2096.                 back[i].r.ssl_con = SSL_new(openssl_ctx);
  2097.                 if (back[i].r.ssl_con) {
  2098.                   SSL_clear(back[i].r.ssl_con);
  2099.                   if (SSL_set_fd(back[i].r.ssl_con, back[i].r.soc) == 1) {
  2100.                     SSL_set_connect_state(back[i].r.ssl_con);
  2101.                     back[i].status = 102;         /* handshake wait */
  2102.                   } else
  2103.                     back[i].r.statuscode=STATUSCODE_SSL_HANDSHAKE;
  2104.                 } else
  2105.                   back[i].r.statuscode=STATUSCODE_SSL_HANDSHAKE;
  2106.               }
  2107.               /* Error */
  2108.               if (back[i].r.statuscode == STATUSCODE_SSL_HANDSHAKE) {
  2109.                 strcpybuff(back[i].r.msg, "bad SSL/TLS handshake");
  2110.                 deletehttp(&back[i].r);
  2111.                 back[i].r.soc=INVALID_SOCKET;
  2112.                 back[i].r.statuscode=STATUSCODE_NON_FATAL;
  2113.                 back[i].status=0;
  2114.                 back_set_finished(sback, i);
  2115.               }
  2116.             }
  2117.             
  2118. #endif
  2119.  
  2120. #if BDEBUG==1
  2121.           printf("..connect ok on socket %d\n",back[i].r.soc);
  2122. #endif
  2123.           
  2124.           if ((back[i].r.soc != INVALID_SOCKET) && (back[i].status==100)) {
  2125.             /* limit nb. connections/seconds to avoid server overload */
  2126.             /*if (opt->maxconn>0) {
  2127.               Sleep(1000/opt->maxconn);
  2128.             }*/
  2129.             
  2130.             back[i].ka_time_start=time_local();
  2131.             if (back[i].timeout>0) {    // refresh timeout si besoin est
  2132.               back[i].timeout_refresh=back[i].ka_time_start;
  2133.             }
  2134.             if (back[i].rateout>0) {    // le taux de transfert de base sur le dΘbut de la connexion
  2135.               back[i].rateout_time=back[i].ka_time_start;
  2136.             }
  2137.             // envoyer header
  2138.             //if (strcmp(back[i].url_sav,BACK_ADD_TEST)!=0)    // vrai get
  2139.             HTS_STAT.stat_nrequests++;
  2140.             if (!back[i].head_request)
  2141.               http_sendhead(opt->cookie,0,back[i].send_too,back[i].url_adr,back[i].url_fil,back[i].referer_adr,back[i].referer_fil,&back[i].r);         
  2142.             else if (back[i].head_request==2)  // test en GET!
  2143.               http_sendhead(opt->cookie,0,back[i].send_too,back[i].url_adr,back[i].url_fil,back[i].referer_adr,back[i].referer_fil,&back[i].r);         
  2144.             else        // test!
  2145.               http_sendhead(opt->cookie,1,back[i].send_too,back[i].url_adr,back[i].url_fil,back[i].referer_adr,back[i].referer_fil,&back[i].r);         
  2146.             back[i].status=99;  // attendre en tΩte maintenant
  2147.           }
  2148.         }
  2149.         
  2150.         // attente gethostbyname
  2151.       }
  2152. #if HTS_USEOPENSSL
  2153.       else if (SSL_is_available && back[i].status==102) {   // wait for SSL handshake
  2154.         /* SSL mode */
  2155.         if (back[i].r.ssl) {
  2156.           int conn_code;
  2157.           if ((conn_code = SSL_connect(back[i].r.ssl_con)) <= 0) {
  2158.             /* non blocking I/O, will retry */
  2159.             int err_code = SSL_get_error(back[i].r.ssl_con, conn_code);
  2160.             if (
  2161.               (err_code != SSL_ERROR_WANT_READ)
  2162.               &&
  2163.               (err_code != SSL_ERROR_WANT_WRITE)
  2164.               ) {
  2165.               char tmp[256];
  2166.               tmp[0]='\0';
  2167.               ERR_error_string(err_code, tmp);
  2168.               back[i].r.msg[0]='\0';
  2169.               strncatbuff(back[i].r.msg, tmp, sizeof(back[i].r.msg) - 2);
  2170.               if (!strnotempty(back[i].r.msg)) {
  2171.                 sprintf(back[i].r.msg, "SSL/TLS error %d", err_code);
  2172.               }
  2173.               deletehttp(&back[i].r);
  2174.               back[i].r.soc=INVALID_SOCKET;
  2175.               back[i].r.statuscode=STATUSCODE_NON_FATAL;
  2176.               back[i].status=0;
  2177.               back_set_finished(sback, i);
  2178.               }
  2179.           } else {        /* got it! */
  2180.             back[i].status=100;       // back to waitconnect
  2181.           }
  2182.         } else {
  2183.           strcpybuff(back[i].r.msg, "unexpected SSL/TLS error");
  2184.           deletehttp(&back[i].r);
  2185.           back[i].r.soc=INVALID_SOCKET;
  2186.           back[i].r.statuscode=STATUSCODE_NON_FATAL;
  2187.           back[i].status=0;
  2188.           back_set_finished(sback, i);
  2189.         }
  2190.         
  2191.       }
  2192. #endif
  2193. #if HTS_XGETHOST
  2194.       else if (back[i].status==101) {  // attendre gethostbyname
  2195. #if DEBUGDNS 
  2196.         //printf("status 101 for %s\n",back[i].url_adr);
  2197. #endif
  2198.  
  2199.         if (!gestion_timeout)
  2200.           if (back[i].timeout>0)
  2201.             gestion_timeout=1;
  2202.  
  2203.         if (host_wait(&back[i])) {    // prΩt
  2204.           back[i].status=100;        // attente connexion
  2205.           if (back[i].timeout>0) {    // refresh timeout si besoin est
  2206.             back[i].timeout_refresh=time_local();
  2207.           }
  2208.           if (back[i].rateout>0) {    // le taux de transfert de base sur le dΘbut de la connexion
  2209.             back[i].rateout_time=time_local();
  2210.           }
  2211.  
  2212.           back[i].r.soc=http_xfopen(0,0,0,back[i].send_too,back[i].url_adr,back[i].url_fil,&(back[i].r));
  2213.           if (back[i].r.soc==INVALID_SOCKET) {
  2214.             back[i].status=0;  // fini, erreur
  2215.             back_set_finished(sback, i);
  2216.             if (back[i].r.soc!=INVALID_SOCKET) {
  2217. #if HTS_DEBUG_CLOSESOCK
  2218.               DEBUG_W("back_wait(2): deletehttp\n");
  2219. #endif
  2220.               deletehttp(&back[i].r);
  2221.             }
  2222.             back[i].r.soc=INVALID_SOCKET;
  2223.             back[i].r.statuscode=STATUSCODE_NON_FATAL;
  2224.             if (strnotempty(back[i].r.msg)==0) 
  2225.               strcpybuff(back[i].r.msg,"Unable to resolve host name");
  2226.           }
  2227.         }
  2228.         
  2229.  
  2230.       // ---- FLAG READ MIS A UN?: POUR LA RECEPTION
  2231.       }
  2232. #endif
  2233. #if USE_BEGINTHREAD
  2234.       // ..rien α faire, c'est magic les threads
  2235. #else
  2236.       else if (back[i].status==1000) {  // en rΘception ftp
  2237.         if (!fexist(back[i].location_buffer)) {    // terminΘ
  2238.           FILE* fp;
  2239.           fp=fopen(fconcat(back[i].location_buffer,".ok"),"rb");
  2240.           if (fp) {
  2241.             int j=0;
  2242.             fscanf(fp,"%d ",&(back[i].r.statuscode));
  2243.             while(!feof(fp)) {
  2244.               int c = fgetc(fp);
  2245.               if (c!=EOF)
  2246.                 back[i].r.msg[j++]=c;
  2247.             }
  2248.             back[i].r.msg[j++]='\0';
  2249.             fclose(fp);
  2250.             unlink(fconcat(back[i].location_buffer,".ok"));
  2251.             strcpybuff(fconcat(back[i].location_buffer,".ok"),"");
  2252.           } else {
  2253.             strcpybuff(back[i].r.msg,"Unknown ftp result, check if file is ok");
  2254.             back[i].r.statuscode=STATUSCODE_INVALID;
  2255.           }
  2256.           back[i].status=0;
  2257.           back_set_finished(sback, i);
  2258.           // finalize transfer
  2259.           if (back[i].r.statuscode>0) {
  2260.             back_finalize(opt,cache,sback,i);
  2261.           }
  2262.         }
  2263.       }
  2264. #endif
  2265.       else if (back[i].status==1001) {  // ftp ready
  2266.         back[i].status=0;
  2267.         back_set_finished(sback, i);
  2268.         // finalize transfer
  2269.         if (back[i].r.statuscode>0) {
  2270.           back_finalize(opt,cache,sback,i);
  2271.         }
  2272.       }
  2273.       else if ((back[i].status>0) && (back[i].status<1000)) {  // en rΘception http
  2274.         int dispo=0;
  2275.         
  2276.         // vΘrifier l'existance de timeout-check
  2277.         if (!gestion_timeout)
  2278.           if (back[i].timeout>0)
  2279.             gestion_timeout=1;
  2280.           
  2281.           // donnΘes dispo?
  2282.           //## if (back[i].url_adr[0]!=lOCAL_CHAR)
  2283.           if (back[i].r.is_file)
  2284.             dispo=1;
  2285.           else if (back[i].r.ssl)
  2286.             dispo=1;
  2287.           else
  2288.             dispo=FD_ISSET(back[i].r.soc,&fds);
  2289.  
  2290.           // Check transfer rate!
  2291.           if (!max_read_bytes)
  2292.             dispo=0;                // limit transfer rate
  2293.           
  2294.           if (dispo) {    // donnΘes dispo
  2295.             LLint retour_fread;
  2296.             busy_recv=1;    // on rΘcupΦre encore
  2297. #if BDEBUG==1
  2298.             printf("..data available on socket %d\n",back[i].r.soc);
  2299. #endif
  2300.  
  2301.             
  2302.             // range size hack old location
  2303.  
  2304. #if HTS_DIRECTDISK
  2305.             // Court-circuit:
  2306.             // Peut-on stocker le fichier directement sur disque?
  2307.             // Ahh que ca serait vachement mieux et que ahh que la mΘmoire vous dit merci!
  2308.             if (back[i].status) {
  2309.               if (back[i].r.is_write==0) {  // mode mΘmoire
  2310.                 if (back[i].r.adr==NULL) {  // rien n'a ΘtΘ Θcrit
  2311.                   if (!back[i].testmode) {  // pas mode test
  2312.                     if (strnotempty(back[i].url_sav)) {
  2313.                       if (strcmp(back[i].url_fil,"/robots.txt")) {
  2314.                         if (back[i].r.statuscode==200) {  // 'OK'
  2315.                           if (!is_hypertext_mime(back[i].r.contenttype, back[i].url_fil)) {    // pas HTML
  2316.                             if (opt->getmode&2) {    // on peut ecrire des non html
  2317.                               int fcheck=0;
  2318.                               back[i].r.is_write=1;    // Θcrire
  2319.                               if (back[i].r.compressed
  2320.                                 &&
  2321.                                 /* .gz are *NOT* depacked!! */
  2322.                                 (strfield(get_ext(back[i].url_sav),"gz") == 0)
  2323.                                 ) {
  2324.                                 back[i].tmpfile_buffer[0]='\0';
  2325.                                 back[i].tmpfile=tmpnam(back[i].tmpfile_buffer);
  2326.                                 if (back[i].tmpfile != NULL && back[i].tmpfile[0])
  2327.                                   back[i].r.out=fopen(back[i].tmpfile,"wb");
  2328.                               } else {
  2329.                                 file_notify(back[i].url_adr, back[i].url_fil, back[i].url_sav, 1, 1, back[i].r.notmodified);
  2330.                                 back[i].r.compressed=0;
  2331.                                 back[i].r.out=filecreate(back[i].url_sav);
  2332.                               }
  2333.                               if (back[i].r.out==NULL) {
  2334.                                 if ((fcheck=check_fatal_io_errno())) {
  2335.                                                                     fspc(opt->log,"error"); fprintf(opt->log,"Mirror aborted: disk full or filesystem problems"LF); test_flush;
  2336.                                   opt->state.exit_xh=-1;   /* fatal error */
  2337.                                 }
  2338.                               }
  2339. #if HDEBUG
  2340.                               printf("direct-disk: %s\n",back[i].url_sav);
  2341. #endif
  2342.                               if ((opt->debug>1) && (opt->log!=NULL)) {
  2343.                                 fspc(opt->log,"debug"); fprintf(opt->log,"File received from net to disk: %s%s"LF,back[i].url_adr,back[i].url_fil); test_flush;
  2344.                               }
  2345.                               
  2346.                               if (back[i].r.out==NULL) {
  2347.                                 if (opt->errlog) {
  2348.                                   fspc(opt->errlog,"error");
  2349.                                   fprintf(opt->errlog,"Unable to save file %s : %s"LF,back[i].url_sav, strerror(errno));
  2350.                                   if (fcheck) {
  2351.                                     fspc(opt->errlog,"error");
  2352.                                     fprintf(opt->errlog,"* * Fatal write error, giving up"LF);
  2353.                                   }
  2354.                                   test_flush;
  2355.                                 }
  2356.                                 back[i].r.is_write=0;    // erreur, abandonner
  2357. #if HDEBUG
  2358.                                 printf("..error!\n");
  2359. #endif
  2360.                               }
  2361. #if HTS_WIN==0
  2362.                               else chmod(back[i].url_sav,HTS_ACCESS_FILE);      
  2363. #endif          
  2364.                             } else {  // on coupe tout!
  2365.                               if ((opt->debug>1) && (opt->log!=NULL)) {
  2366.                                 fspc(opt->log,"debug"); fprintf(opt->log,"File cancelled (non HTML): %s%s"LF,back[i].url_adr,back[i].url_fil); test_flush;
  2367.                               }
  2368.                               back[i].status=0;  // terminΘ
  2369.                               back_set_finished(sback, i);
  2370.                               if (!back[i].testmode)
  2371.                                 back[i].r.statuscode=STATUSCODE_INVALID;    // EUHH CANCEL
  2372.                               else
  2373.                                 back[i].r.statuscode=STATUSCODE_TEST_OK;    // "TEST OK"
  2374.                               if (back[i].r.soc!=INVALID_SOCKET) {
  2375. #if HTS_DEBUG_CLOSESOCK
  2376.                                 DEBUG_W("back_wait(3): deletehttp\n");
  2377. #endif
  2378.                                 deletehttp(&back[i].r);
  2379.                               }
  2380.                               back[i].r.soc=INVALID_SOCKET;
  2381.                             }
  2382.                           }
  2383.                         }
  2384.                       }
  2385.                     }
  2386.                   }
  2387.                 }
  2388.               }
  2389.             }
  2390. #endif              
  2391.  
  2392.             // rΘception de donnΘes depuis socket ou fichier
  2393.             if (back[i].status) {
  2394.               if (back[i].status==99)  // recevoir par bloc de lignes
  2395.                 retour_fread=http_xfread1(&(back[i].r),0);
  2396.               else if (back[i].status==98 || back[i].status==97) { // recevoir longueur chunk en hexa caractΦre par caractΦre
  2397.                 // backuper pour lire dans le buffer chunk
  2398.                 htsblk r;
  2399.                 memcpy(&r, &(back[i].r), sizeof(htsblk));
  2400.                 back[i].r.is_write=0;                   // mΘmoire
  2401.                 back[i].r.adr=back[i].chunk_adr;        // adresse
  2402.                 back[i].r.size=back[i].chunk_size;      // taille taille chunk
  2403.                 back[i].r.totalsize=-1;                 // total inconnu
  2404.                 back[i].r.out=NULL;
  2405.                 back[i].r.is_file=0;
  2406.                 //
  2407.                 // ligne par ligne
  2408.                 retour_fread=http_xfread1(&(back[i].r),-1);
  2409.                 // modifier et restaurer
  2410.                 back[i].chunk_adr=back[i].r.adr;        // adresse
  2411.                 back[i].chunk_size=back[i].r.size;      // taille taille chunk
  2412.                 memcpy(&(back[i].r), &r, sizeof(htsblk));    // restaurer vΘritable r
  2413.               }
  2414.               else if (back[i].is_chunk) {         // attention chunk, limiter taille α lire
  2415. #if CHUNKDEBUG==1
  2416.                 printf("[%d] read %d bytes\n",(int)back[i].r.soc,(int)min(back[i].r.totalsize-back[i].r.size,max_read_bytes));
  2417. #endif
  2418.                 retour_fread=(int) http_xfread1(&(back[i].r),(int) min(back[i].r.totalsize-back[i].r.size,max_read_bytes));
  2419.               } else              
  2420.                 retour_fread=(int) http_xfread1(&(back[i].r),(int) max_read_bytes);
  2421.                 // retour_fread=http_fread1(&(back[i].r));
  2422.             } else
  2423.               retour_fread=READ_EOF;                    // interruption ou annulation interne (peut ne pas Ωtre une erreur)
  2424.             
  2425.             // Si rΘception chunk, tester si on est pas α la fin!
  2426.             if (back[i].status==1) {
  2427.               if (back[i].is_chunk) {     // attendre prochain chunk
  2428.                 if (back[i].r.size==back[i].r.totalsize) {      // fin chunk!
  2429.                   //printf("chunk end at %d\n",back[i].r.size);
  2430.                   back[i].status=97;  /* fetch ending CRLF */
  2431.                   if (back[i].chunk_adr!=NULL) { 
  2432.                     freet(back[i].chunk_adr); 
  2433.                     back[i].chunk_adr=NULL; 
  2434.                   } 
  2435.                   back[i].chunk_size=0;
  2436.                   retour_fread=0;       // pas d'erreur
  2437. #if CHUNKDEBUG==1
  2438.                   printf("[%d] waiting for current chunk CRLF..\n",(int)back[i].r.soc);
  2439. #endif
  2440.                 }
  2441.               } else if (back[i].r.keep_alive) {
  2442.                 if (back[i].r.size==back[i].r.totalsize) {      // fin!
  2443.                   retour_fread=READ_EOF;       // end
  2444.                 }
  2445.               }
  2446.             }
  2447.             
  2448.             if (retour_fread < 0) {    // fin rΘception
  2449.               back[i].status=0;    // terminΘ
  2450.               back_set_finished(sback, i);
  2451.              /*KA back[i].r.soc=INVALID_SOCKET; */
  2452. #if CHUNKDEBUG==1
  2453.               if (back[i].is_chunk)
  2454.                 printf("[%d] must be the last chunk for %s (connection closed) - %d/%d\n",(int)back[i].r.soc,back[i].url_fil,back[i].r.size,back[i].r.totalsize);
  2455. #endif
  2456.               if (retour_fread < 0 && retour_fread != READ_EOF) {
  2457.                 if (back[i].r.size > 0)
  2458.                   strcatbuff(back[i].r.msg, "Interrupted transfer");
  2459.                 else
  2460.                   strcatbuff(back[i].r.msg, "No data (connection closed)");
  2461.                 back[i].r.statuscode=STATUSCODE_CONNERROR;
  2462.               } else if ((back[i].r.statuscode <= 0) && (strnotempty(back[i].r.msg)==0)) {
  2463. #if HDEBUG
  2464.                 printf("error interruped: %s\n",back[i].r.adr);
  2465. #endif        
  2466.                 if (back[i].r.size>0)
  2467.                   strcatbuff(back[i].r.msg,"Interrupted transfer");
  2468.                 else
  2469.                   strcatbuff(back[i].r.msg,"No data (connection closed)");
  2470.                 back[i].r.statuscode=STATUSCODE_CONNERROR;
  2471.               }
  2472.  
  2473.               // Close socket
  2474.               if (back[i].r.soc!=INVALID_SOCKET) {
  2475. #if HTS_DEBUG_CLOSESOCK
  2476.                 DEBUG_W("back_wait(4): deletehttp\n");
  2477. #endif
  2478.                 /*KA deletehttp(&back[i].r);*/
  2479.                 back_maydeletehttp(opt, cache, sback, i);
  2480.               }
  2481.  
  2482.               // finalize transfer
  2483.               if (back[i].r.statuscode>0
  2484.                 && !IS_DELAYED_EXT(back[i].url_sav)
  2485.                 ) {
  2486.                 back_finalize(opt,cache,sback,i);
  2487.               }
  2488.  
  2489.               if (back[i].r.totalsize>0) {    // tester totalsize
  2490.               //if ((back[i].r.totalsize>0) && (back[i].status==99)) {    // tester totalsize
  2491.                 if (back[i].r.totalsize!=back[i].r.size) {  // pas la mΩme!
  2492.                   if (!opt->tolerant) {
  2493.                     //#if HTS_CL_IS_FATAL
  2494.                     deleteaddr(&back[i].r);
  2495.                     if (back[i].r.size<back[i].r.totalsize)
  2496.                       back[i].r.statuscode=STATUSCODE_CONNERROR;        // recatch
  2497.                     sprintf(back[i].r.msg,"Incorrect length ("LLintP" Bytes, "LLintP" expected)",(LLint)back[i].r.size,(LLint)back[i].r.totalsize);
  2498.                   } else {
  2499.                     //#else
  2500.                     // Un warning suffira..
  2501.                     if (cache->errlog!=NULL) {
  2502.                       fspc(cache->errlog,"warning"); fprintf(cache->errlog,"Incorrect length ("LLintP"!="LLintP" expected) for %s%s"LF,(LLint)back[i].r.size,(LLint)back[i].r.totalsize,back[i].url_adr,back[i].url_fil);
  2503.                     }
  2504.                     //#endif
  2505.                   }
  2506.                 }
  2507.               }
  2508. #if BDEBUG==1
  2509.               printf("transfer ok\n");
  2510. #endif
  2511.             } else if (retour_fread > 0) {    // pas d'erreur de rΘception et data
  2512.               if (back[i].timeout>0) {    // refresh timeout si besoin est
  2513.                 back[i].timeout_refresh=time_local();
  2514.               }
  2515.  
  2516.               // Traitement des en tΩtes chunks ou en tΩtes
  2517.               if (back[i].status==98 || back[i].status==97) {        // rΘception taille chunk en hexa (  aprΦs les en tΩtes, peut ne pas
  2518.                 if (back[i].chunk_size > 0 && back[i].chunk_adr[back[i].chunk_size-1]==10) {
  2519.                   int chunk_size=-1;
  2520.                   char chunk_data[64];
  2521.                   if (back[i].chunk_size<32) {      // pas trop gros
  2522.                     char* chstrip=back[i].chunk_adr;
  2523.                     back[i].chunk_adr[ back[i].chunk_size-1]='\0';    // octet nul 
  2524.                     // skip leading spaces or cr
  2525.                     while(isspace(*chstrip)) chstrip++;
  2526.                     chunk_data[0] = '\0';
  2527.                     strncatbuff(chunk_data, chstrip, sizeof(chunk_data) - 2);
  2528.                     // strip chunk-extension
  2529.                     while( (chstrip = strchr(chunk_data, ';'))) *chstrip='\0';
  2530.                     while( (chstrip = strchr(chunk_data, ' '))) *chstrip='\0';
  2531.                     while( (chstrip = strchr(chunk_data, '\r'))) *chstrip='\0';
  2532. #if CHUNKDEBUG==1
  2533.                     printf("[%d] chunk received and read: %s\n",(int)back[i].r.soc,chunk_data);
  2534. #endif
  2535.                     if (back[i].r.totalsize<0)
  2536.                       back[i].r.totalsize=0;        // initialiser α 0
  2537.                     if (back[i].status==98) {   // "real" chunk
  2538.                       if (sscanf(chunk_data,"%x",&chunk_size) == 1) {
  2539.                         if (chunk_size > 0)
  2540.                           back[i].chunk_blocksize = chunk_size;  /* the data block chunk size */
  2541.                                                 else
  2542.                                                     back[i].chunk_blocksize = -1;  /* ending */
  2543.                                                 back[i].r.totalsize+=chunk_size;    // noter taille
  2544.                                                 if (back[i].r.adr != NULL || !back[i].r.is_write) {  // Not to disk
  2545.                                                     back[i].r.adr=(char*) realloct(back[i].r.adr,(INTsys) back[i].r.totalsize + 1);
  2546.                                                     if (!back[i].r.adr) {
  2547.                                                         if (cache->errlog!=NULL) {
  2548.                                                             fprintf(cache->errlog,"Error: Not enough memory ("LLintP") for %s%s"LF,(LLint)back[i].r.totalsize,back[i].url_adr,back[i].url_fil);
  2549.                                                         }
  2550.                                                     }
  2551.                                                 }
  2552. #if CHUNKDEBUG==1
  2553.                         printf("[%d] chunk length: %d - next total "LLintP":\n",(int)back[i].r.soc,(int)chunk_size,(LLint)back[i].r.totalsize);
  2554. #endif
  2555.                       } else {
  2556.                         if (cache->errlog!=NULL) {
  2557.                           fprintf(cache->errlog,"Warning: Illegal chunk (%s) for %s%s"LF,back[i].chunk_adr,back[i].url_adr,back[i].url_fil);
  2558.                         }
  2559.                       }
  2560.                     } else {   /* back[i].status==97 : just receiving ending CRLF after data */
  2561.                       if (chunk_data[0] == '\0') {
  2562.                         if (back[i].chunk_blocksize > 0)
  2563.                           chunk_size=(int)back[i].chunk_blocksize;  /* recent data chunk size */
  2564.                         else if (back[i].chunk_blocksize == -1)
  2565.                           chunk_size=0;                        /* ending chunk */
  2566.                         else
  2567.                           chunk_size=1;                        /* fake positive size for 1st chunk history */
  2568. #if CHUNKDEBUG==1
  2569.                         printf("[%d] chunk CRLF seen\n", (int)back[i].r.soc);
  2570. #endif
  2571.                       } else {
  2572.                         if (cache->errlog!=NULL) {
  2573.                           fprintf(cache->errlog,"Warning: Illegal chunk CRLF (%s) for %s%s"LF,back[i].chunk_adr,back[i].url_adr,back[i].url_fil);
  2574.                         }
  2575. #if CHUNKDEBUG==1
  2576.                         printf("[%d] chunk CRLF ERROR!! : '%s'\n", (int)back[i].r.soc, chunk_data);
  2577. #endif
  2578.                       }
  2579.                     }
  2580.                   } else {                                  
  2581.                     if (cache->errlog!=NULL) {
  2582.                       fprintf(cache->errlog,"Warning: Chunk too big ("LLintP") for %s%s"LF,(LLint)back[i].chunk_size,back[i].url_adr,back[i].url_fil);
  2583.                     }
  2584.                   }
  2585.                     
  2586.                   // ok, continuer sur le body
  2587.                     
  2588.                   // si chunk non nul continuer (ou commencer)
  2589.                   if (back[i].status==97 && chunk_size > 0) {
  2590.                     back[i].status = 98;  /* waiting for next chunk (NN\r\n<data>\r\nNN\r\n<data>..\r\n0\r\n\r\n) */
  2591. #if CHUNKDEBUG==1
  2592.                     printf("[%d] waiting for next chunk\n", (int)back[i].r.soc);
  2593. #endif
  2594.                   } else if (back[i].status==98 && chunk_size == 0) {  /* final chunk */
  2595.                     back[i].status=97;  /* final CRLF */
  2596. #if CHUNKDEBUG==1
  2597.                     printf("[%d] waiting for final CRLF (chunk)\n", (int)back[i].r.soc);
  2598. #endif
  2599.                   } else if (back[i].status==98 && chunk_size >= 0) {  /* will fetch data now */
  2600.                     back[i].status=1;     // continuer body    
  2601. #if CHUNKDEBUG==1
  2602.                     printf("[%d] waiting for body (chunk)\n", (int)back[i].r.soc);
  2603. #endif
  2604.                   } else {                /* zero-size-chunk-CRLF (end) or error */
  2605. #if CHUNKDEBUG==1
  2606.                     printf("[%d] chunk end, total: %d\n",(int)back[i].r.soc,back[i].r.size);
  2607. #endif
  2608.                     /* End */
  2609.                     //if (back[i].status==97) {
  2610.                     back[i].status=0;     // fin  
  2611.                     back_set_finished(sback, i);
  2612.                     //}
  2613.  
  2614.                     // finalize transfer if not temporary
  2615.                     if (!IS_DELAYED_EXT(back[i].url_sav)) {
  2616.                       back_finalize(opt,cache,sback,i);
  2617.                     } else {
  2618.                       if (back[i].r.statuscode == 200) {
  2619.                         if (cache->errlog!=NULL) {
  2620.                           fspc(cache->errlog,"warning"); fprintf(cache->errlog,"Unexpected incomplete type with 200 code at %s%s"LF, back[i].url_adr, back[i].url_fil);
  2621.                         }
  2622.                       }
  2623.                     }
  2624.                     if (back[i].r.soc!=INVALID_SOCKET) {
  2625. #if HTS_DEBUG_CLOSESOCK
  2626.                       DEBUG_W("back_wait(5): deletehttp\n");
  2627. #endif
  2628.                       /* Error */
  2629.                       if (chunk_size < 0) {
  2630.                         deletehttp(&back[i].r); back[i].r.soc=INVALID_SOCKET;
  2631.                         deleteaddr(&back[i].r);
  2632.                         back[i].r.statuscode=STATUSCODE_INVALID;
  2633.                         strcpybuff(back[i].r.msg,"Invalid chunk");
  2634. #if CHUNKDEBUG==1
  2635.                         printf("[%d] chunk error\n", (int)back[i].r.soc);
  2636. #endif
  2637.                       } else /* if chunk_size == 0 */ {
  2638. #if CHUNKDEBUG==1
  2639.                         printf("[%d] all chunks now received\n", (int)back[i].r.soc);
  2640. #endif
  2641.                           
  2642.                         /* Tester totalsize en fin de chunk */
  2643.                         if ((back[i].r.totalsize>0)) {    // tester totalsize
  2644.                           if (back[i].r.totalsize!=back[i].r.size) {  // pas la mΩme!
  2645.                             if (!opt->tolerant) {
  2646.                               deleteaddr(&back[i].r);
  2647.                               back[i].r.statuscode=STATUSCODE_INVALID;
  2648.                               strcpybuff(back[i].r.msg,"Incorrect length");
  2649.                             } else {
  2650.                               // Un warning suffira..
  2651.                               if (cache->errlog!=NULL) {
  2652.                                 fspc(cache->errlog,"warning"); fprintf(cache->errlog,"Incorrect length ("LLintP"!="LLintP" expected) for %s%s"LF,(LLint)back[i].r.size,(LLint)back[i].r.totalsize,back[i].url_adr,back[i].url_fil);
  2653.                               }
  2654.                             }
  2655.                           }
  2656.                         }
  2657.                           
  2658.                         /* Oops, trailers! */
  2659.                         if (back[i].r.keep_alive_trailers) {
  2660.                           /* fixme (not yet supported) */
  2661.                         }
  2662.                           
  2663.                       }
  2664.                         
  2665.                         
  2666.                     }
  2667.                   }
  2668.                     
  2669.                   // effacer buffer (chunk en tete)
  2670.                   if (back[i].chunk_adr!=NULL) {
  2671.                     freet(back[i].chunk_adr);
  2672.                     back[i].chunk_adr=NULL;
  2673.                     back[i].chunk_size=0;
  2674.                     // NO! xxback[i].chunk_blocksize = 0;
  2675.                   }
  2676.                   
  2677.                 } // taille buffer chunk > 1 && LF
  2678.                 //
  2679.               } else if (back[i].status==99) {        // en tΩtes (avant le chunk si il est prΘsent)
  2680.                 //
  2681.                 if (back[i].r.size>=2) {
  2682.                   // double LF
  2683.                   if (
  2684.                     ((back[i].r.adr[back[i].r.size-1]==10) && (back[i].r.adr[back[i].r.size-2]==10)) 
  2685.                     ||
  2686.                     (back[i].r.adr[0] == '<')    /* bogus server */
  2687.                     ) {
  2688.                     char rcvd[2048];
  2689.                     int ptr=0;
  2690.                     int noFreebuff=0;
  2691.                     
  2692. #if BDEBUG==1
  2693.                     printf("..ok, header received\n");
  2694. #endif
  2695.                     
  2696.                     
  2697.                     /* Hack for zero-length headers */
  2698.                     if (back[i].status != 0 && back[i].r.adr[0] != '<') {
  2699.                       
  2700.                       // ----------------------------------------
  2701.                       // traiter en-tΩte!
  2702.                       // status-line α rΘcupΘrer
  2703.                       ptr+=binput(back[i].r.adr+ptr,rcvd,2000);
  2704.                       if (strnotempty(rcvd)==0) {
  2705.                         /* Bogus CRLF, OR recycled connection and trailing chunk CRLF */
  2706.                         ptr+=binput(back[i].r.adr+ptr,rcvd,2000);
  2707.                       }
  2708.                       
  2709.                       // traiter status-line
  2710.                       treatfirstline(&back[i].r,rcvd);
  2711.                       
  2712. #if HDEBUG
  2713.                       printf("(Buffer) Status-Code=%d\n",back[i].r.statuscode);
  2714. #endif
  2715.                       if (_DEBUG_HEAD) {
  2716.                         if (ioinfo) {
  2717.                           fprintf(ioinfo,"[%d] response for %s%s:\r\ncode=%d\r\n",
  2718.                             back[i].r.debugid, jump_identification(back[i].url_adr),back[i].url_fil,back[i].r.statuscode);
  2719.                           fprintfio(ioinfo,back[i].r.adr,">>> ");
  2720.                           fprintf(ioinfo,"\r\n");
  2721.                           fflush(ioinfo);
  2722.                         }                    // en-tΩte
  2723.                       }
  2724.                       
  2725.                       // header // ** !attention! HTTP/0.9 non supportΘ
  2726.                       do {
  2727.                         ptr+=binput(back[i].r.adr+ptr,rcvd,2000);          
  2728. #if HDEBUG
  2729.                         printf("(buffer)>%s\n",rcvd);      
  2730. #endif
  2731.                         /*
  2732.                         if (_DEBUG_HEAD) {
  2733.                         if (ioinfo) {
  2734.                         fprintf(ioinfo,"(buffer)>%s\r\n",rcvd);      
  2735.                         fflush(ioinfo);
  2736.                         }
  2737.                         }
  2738.                         */
  2739.                         
  2740.                         if (strnotempty(rcvd))
  2741.                           treathead(opt->cookie,back[i].url_adr,back[i].url_fil,&back[i].r,rcvd);  // traiter
  2742.                         
  2743.                         // parfois les serveurs buggΘs renvoient un content-range avec un 200
  2744.                         if (back[i].r.statuscode==200)  // 'OK'
  2745.                           if (strfield(rcvd,"content-range:"))  // Avec un content-range: relisez les RFC..
  2746.                             back[i].r.statuscode=206;    // FORCER A 206 !!!!!
  2747.                           
  2748.                       } while(strnotempty(rcvd));
  2749.                       // ----------------------------------------                    
  2750.  
  2751.                     } else {
  2752.                       // assume text/html, OK
  2753.                       treatfirstline(&back[i].r, back[i].r.adr);
  2754.                       noFreebuff=1;
  2755.                     }
  2756.                       
  2757.                     // Callback
  2758. #if HTS_ANALYSTE
  2759.                     if (hts_htmlcheck_receivehead != NULL) {
  2760.                       int test_head=hts_htmlcheck_receivehead(back[i].r.adr, back[i].url_adr, back[i].url_fil, back[i].referer_adr, back[i].referer_fil, &back[i].r);
  2761.                       if (test_head!=1) {
  2762.                         if ((opt->debug>0) && (opt->log!=NULL)) {
  2763.                           fspc(opt->log,"warning"); fprintf(opt->log,"External wrapper aborted transfer, breaking connection: %s%s"LF,back[i].url_adr,back[i].url_fil); test_flush;
  2764.                         }
  2765.                         back[i].status=0;  // FINI
  2766.                         back_set_finished(sback, i);
  2767.                         deletehttp(&back[i].r); back[i].r.soc=INVALID_SOCKET;
  2768.                         strcpybuff(back[i].r.msg,"External wrapper aborted transfer");
  2769.                         back[i].r.statuscode = STATUSCODE_INVALID;
  2770.                       }
  2771.                     }
  2772. #endif
  2773.  
  2774.                     // Free headers memory now
  2775.                     // Actually, save them for informational purpose
  2776.                     if (!noFreebuff) {
  2777.                       char* block = back[i].r.adr;
  2778.                       back[i].r.adr = NULL;
  2779.                       deleteaddr(&back[i].r);
  2780.                       back[i].r.headers = block;
  2781.                     }                  
  2782.                     
  2783.                     /* 
  2784.                     Status code and header-response hacks
  2785.                     */
  2786.  
  2787.                     
  2788.                     // Check response : 203 == 200
  2789.                     if (back[i].r.statuscode==203) {  // 'Non-Authoritative Information'
  2790.                       back[i].r.statuscode=200;       // forcer "OK"
  2791.                     } else if (back[i].r.statuscode == 100) {
  2792.                       back[i].status=99;
  2793.                       back[i].r.size=0;
  2794.                       back[i].r.totalsize=0;
  2795.                       back[i].chunk_size=0;
  2796.                       back[i].r.statuscode=STATUSCODE_INVALID;
  2797.                       back[i].r.msg[0]='\0';
  2798.                       if ((opt->debug>1) && (opt->log!=NULL)) {
  2799.                         fspc(opt->log,"debug"); fprintf(opt->log,"Status 100 detected for %s%s, continuing headers"LF,back[i].url_adr,back[i].url_fil); test_flush;
  2800.                       }
  2801.                       continue;
  2802.                     }
  2803.                     
  2804.                     /*
  2805.                     Solve "false" 416 problems
  2806.                     */
  2807.                     if (back[i].r.statuscode==416) {  // 'Requested Range Not Satisfiable'
  2808.                       // Example:
  2809.                       // Range: bytes=2830-
  2810.                       // ->
  2811.                       // Content-Range: bytes */2830
  2812.                       if (back[i].range_req_size == back[i].r.crange) {
  2813.                         filenote(back[i].url_sav,NULL);
  2814.                         file_notify(back[i].url_adr, back[i].url_fil, back[i].url_sav, 0, 0, back[i].r.notmodified);
  2815.                         deletehttp(&back[i].r); back[i].r.soc=INVALID_SOCKET;
  2816.                         back[i].status=0;    // READY
  2817.                         back_set_finished(sback, i);
  2818.                         back[i].r.size=back[i].r.totalsize=back[i].range_req_size;
  2819.                         back[i].r.statuscode=304;     // NOT MODIFIED
  2820.                         if ((opt->debug>1) && (opt->log!=NULL)) {
  2821.                           fspc(opt->log,"debug"); fprintf(opt->log,"File seems complete (good 416 message), breaking connection: %s%s"LF,back[i].url_adr,back[i].url_fil); test_flush;
  2822.                         }
  2823.                       }
  2824.                     }
  2825.                     
  2826.                     // transform 406 into 200 ; we'll catch embedded links inside the choice page
  2827.                     if (back[i].r.statuscode==406) {  // 'Not Acceptable'
  2828.                       back[i].r.statuscode=200;
  2829.                     }
  2830.  
  2831.                                         // 'do not erase already downloaded file'
  2832.                                         // on an updated file
  2833.                                         // with an error : consider a 304 error
  2834.                                         if (!opt->delete_old) {
  2835.                                             if (HTTP_IS_ERROR(back[i].r.statuscode) && back[i].is_update && !back[i].testmode) {
  2836.                                                 if (back[i].url_sav[0] && fexist(back[i].url_sav)) {
  2837.                                                     if ((opt->debug>1) && (opt->log!=NULL)) {
  2838.                                                         fspc(opt->log,"debug"); fprintf(opt->log,"Error ignored %d (%s) because of 'no purge' option for %s%s"LF,back[i].r.statuscode,back[i].r.msg,back[i].url_adr,back[i].url_fil); test_flush;
  2839.                                                     }
  2840.                                                     back[i].r.statuscode = 304;
  2841.                                                     deletehttp(&back[i].r); back[i].r.soc=INVALID_SOCKET;
  2842.                                                 }
  2843.                                             }
  2844.                                         }
  2845.  
  2846.                                         // Various hacks to limit re-transfers when updating a mirror
  2847.                     // Force update if same size detected
  2848.                     if (opt->sizehack) {
  2849.                       // We already have the file
  2850.                       // and ask the remote server for an update
  2851.                       // Some servers, especially dynamic pages severs, always
  2852.                       // answer that the page has been modified since last visit
  2853.                       // And answer with a 200 (OK) response, and the same page
  2854.                       // If the size is the same, and the option has been set, we assume
  2855.                       // that the file is identical - and therefore let's break the connection
  2856.                       if (back[i].is_update) {          // mise α jour
  2857.                         if (back[i].r.statuscode==200 && !back[i].testmode) {  // 'OK'
  2858.                           htsblk r = cache_read(opt,cache,back[i].url_adr,back[i].url_fil,NULL,NULL);    // lire entrΘe cache
  2859.                           if (r.statuscode == 200) {  // OK pas d'erreur cache
  2860.                             LLint len1,len2;
  2861.                             len1=r.totalsize;
  2862.                             len2=back[i].r.totalsize;
  2863.                             if (r.size>0)
  2864.                               len1=r.size;
  2865.                             if (len1>0) {
  2866.                               if (len1 == len2) {             // tailles identiques
  2867.                                 back[i].r.statuscode=304;     // forcer NOT MODIFIED
  2868.                                 deletehttp(&back[i].r); back[i].r.soc=INVALID_SOCKET;
  2869.                                 if ((opt->debug>1) && (opt->log!=NULL)) {
  2870.                                   fspc(opt->log,"debug"); fprintf(opt->log,"File seems complete (same size), breaking connection: %s%s"LF,back[i].url_adr,back[i].url_fil); test_flush;
  2871.                                 }
  2872.                               }
  2873.                             }
  2874.                           } else {
  2875.                             if (opt->errlog!=NULL) {
  2876.                               fspc(opt->errlog,"warning"); fprintf(opt->errlog,"File seems complete (same size), but there was a cache read error: %s%s"LF,back[i].url_adr,back[i].url_fil); test_flush;
  2877.                             }
  2878.                           }
  2879.                           if (r.adr) {
  2880.                             freet(r.adr);
  2881.                                                         r.adr = NULL;
  2882.                           }
  2883.                         }
  2884.                       }
  2885.                     }
  2886.                     
  2887.                     // Various hacks to limit re-transfers when updating a mirror
  2888.                     // Detect already downloaded file (with another browser, for example)
  2889.                     if (opt->sizehack) {
  2890.                       if (!back[i].is_update) {          // mise α jour
  2891.                         if (back[i].r.statuscode==200 && !back[i].testmode) {  // 'OK'
  2892.                           if (!is_hypertext_mime(back[i].r.contenttype, back[i].url_fil)) {    // not HTML
  2893.                             if (strnotempty(back[i].url_sav)) {  // target found
  2894.                               int size = fsize(back[i].url_sav);  // target size
  2895.                               if (size >= 0) {
  2896.                                 if (back[i].r.totalsize == size) {  // same size!
  2897.                                   deletehttp(&back[i].r); back[i].r.soc=INVALID_SOCKET;
  2898.                                   back[i].status=0;    // READY
  2899.                                   back_set_finished(sback, i);
  2900.                                   back[i].r.size=back[i].r.totalsize;
  2901.                                   filenote(back[i].url_sav,NULL);
  2902.                                   file_notify(back[i].url_adr, back[i].url_fil, back[i].url_sav, 0, 0, back[i].r.notmodified);
  2903.                                   back[i].r.statuscode=304;     // NOT MODIFIED
  2904.                                   if ((opt->debug>1) && (opt->log!=NULL)) {
  2905.                                     fspc(opt->log,"debug"); fprintf(opt->log,"File seems complete (same size file discovered), breaking connection: %s%s"LF,back[i].url_adr,back[i].url_fil); test_flush;
  2906.                                   }
  2907.                                 }
  2908.                               }
  2909.                             }
  2910.                           }
  2911.                         }
  2912.                       }
  2913.                     }
  2914.                     
  2915.                     // Various hacks to limit re-transfers when updating a mirror
  2916.                     // Detect bad range: header
  2917.                     if (opt->sizehack) {
  2918.                       // We have request for a partial file (with a 'Range: NNN-' header)
  2919.                       // and received a complete file notification (200), with 'Content-length: NNN'
  2920.                       // it might be possible that we had the complete file
  2921.                       // this is the case in *most* cases, so break the connection
  2922.                       if (back[i].r.is_write==0) {  // mode mΘmoire
  2923.                         if (back[i].r.adr==NULL) {  // rien n'a ΘtΘ Θcrit
  2924.                           if (!back[i].testmode) {  // pas mode test
  2925.                             if (strnotempty(back[i].url_sav)) {
  2926.                               if (strcmp(back[i].url_fil,"/robots.txt")) {
  2927.                                 if (back[i].r.statuscode==200) {  // 'OK'
  2928.                                   if (!is_hypertext_mime(back[i].r.contenttype, back[i].url_fil)) {    // pas HTML
  2929.                                     if (back[i].r.statuscode==200) {      // "OK"
  2930.                                       if (back[i].range_req_size>0) {     // but Range: requested
  2931.                                         if (back[i].range_req_size == back[i].r.totalsize) {    // And same size
  2932. #if HTS_DEBUG_CLOSESOCK
  2933.                                           DEBUG_W("back_wait(skip_range): deletehttp\n");
  2934. #endif
  2935.                                           deletehttp(&back[i].r); back[i].r.soc=INVALID_SOCKET;
  2936.                                           back[i].status=0;    // READY
  2937.                                           back_set_finished(sback, i);
  2938.                                           back[i].r.size=back[i].r.totalsize;
  2939.                                           filenote(back[i].url_sav,NULL);
  2940.                                           file_notify(back[i].url_adr, back[i].url_fil, back[i].url_sav, 0, 0, back[i].r.notmodified);
  2941.                                           back[i].r.statuscode=304;     // NOT MODIFIED
  2942.                                           if ((opt->debug>1) && (opt->log!=NULL)) {
  2943.                                             fspc(opt->log,"debug"); fprintf(opt->log,"File seems complete (reget failed), breaking connection: %s%s"LF,back[i].url_adr,back[i].url_fil); test_flush;
  2944.                                           }
  2945.                                         }
  2946.                                       }
  2947.                                     }
  2948.                                     
  2949.                                   }
  2950.                                 }
  2951.                               }
  2952.                             }
  2953.                           }
  2954.                         }
  2955.                       }
  2956.                     }
  2957.                     // END - Various hacks to limit re-transfers when updating a mirror
  2958.  
  2959.                     /* 
  2960.                     End of status code and header-response hacks
  2961.                     */
  2962.  
  2963.                     
  2964.                     
  2965.                     /* Interdiction taille par le wizard? */
  2966.                     if (back[i].r.soc!=INVALID_SOCKET) {
  2967.                       if (!back_checksize(opt,&back[i],1)) {
  2968.                         back[i].status=0;  // FINI
  2969.                         back_set_finished(sback, i);
  2970.                         back[i].r.statuscode=STATUSCODE_TOO_BIG;
  2971.                         deletehttp(&back[i].r); back[i].r.soc=INVALID_SOCKET;
  2972.                         if (!back[i].testmode)
  2973.                           strcpybuff(back[i].r.msg,"File too big");
  2974.                         else
  2975.                           strcpybuff(back[i].r.msg,"Test: File too big");
  2976.                       }
  2977.                     }
  2978.                     
  2979.                     /* sinon, continuer */
  2980.                     /* if (back[i].r.soc!=INVALID_SOCKET) {   // ok rΘcupΘrer body? */
  2981.                     // head: terminΘ
  2982.                     if (back[i].head_request) {
  2983.                       if ((opt->debug>1) && (opt->log!=NULL)) {
  2984.                         fspc(opt->log,"debug"); fprintf(opt->log,"Tested file: %s%s"LF,back[i].url_adr,back[i].url_fil); test_flush;
  2985.                       }
  2986. #if HTS_DEBUG_CLOSESOCK
  2987.                       DEBUG_W("back_wait(head request): deletehttp\n");
  2988. #endif
  2989.                       // Couper connexion
  2990.                       if (!back[i].http11) {    /* NO KA */
  2991.                         deletehttp(&back[i].r); back[i].r.soc=INVALID_SOCKET;
  2992.                       }
  2993.                       back[i].status=0;  // terminΘ
  2994.                       back_set_finished(sback, i);
  2995.                     }
  2996.                     // traiter une Θventuelle erreur 304 (cache α jour utilisable)
  2997.                     else if (back[i].r.statuscode==304) {  // document α jour dans le cache
  2998.                       // lire dans le cache
  2999.                       // ** NOTE: pas de vΘrif de la taille ici!!
  3000. #if HTS_DEBUG_CLOSESOCK
  3001.                       DEBUG_W("back_wait(file is not modified): deletehttp\n");
  3002. #endif
  3003.                       /* clear everything but connection: switch, close, and reswitch */
  3004.                       {
  3005.                         htsblk tmp;
  3006.                         memset(&tmp, 0, sizeof(tmp));
  3007.                         back_connxfr(&back[i].r, &tmp);
  3008.                         back[i].r=cache_read(opt,cache,back[i].url_adr,back[i].url_fil,back[i].url_sav,back[i].location_buffer);
  3009.                         back[i].r.location=back[i].location_buffer;
  3010.                         back_connxfr(&tmp,&back[i].r);
  3011.                       }
  3012.  
  3013.                       // hack:
  3014.                       // In case of 'if-unmodified-since' hack, a 304 status can be sent
  3015.                       // then, force 'ok' status
  3016.                       if (back[i].r.statuscode == STATUSCODE_INVALID) {
  3017.                         if (fexist(back[i].url_sav)) {
  3018.                           back[i].r.statuscode=200;     // OK
  3019.                           strcpybuff(back[i].r.msg, "OK (cached)");
  3020.                           back[i].r.is_file=1;
  3021.                           back[i].r.totalsize = back[i].r.size = fsize(back[i].url_sav);
  3022.                           get_httptype(back[i].r.contenttype, back[i].url_sav, 1);
  3023.                           if ((opt->debug>0) && (opt->log!=NULL)) {
  3024.                             fspc(opt->log,"debug"); fprintf(opt->log,"Not-modified status without cache guessed: %s%s"LF,back[i].url_adr,back[i].url_fil); test_flush;
  3025.                           }
  3026.                         }
  3027.                       }
  3028.  
  3029.                       // Status is okay?
  3030.                       if (back[i].r.statuscode!=-1) { // pas d'erreur de lecture
  3031.                         back[i].status=0;         // OK prΩt
  3032.                         back_set_finished(sback, i);
  3033.                         back[i].r.notmodified=1;  // NON modifiΘ!
  3034.                         if ((opt->debug>0) && (opt->log!=NULL)) {
  3035.                           fspc(opt->log,"debug"); fprintf(opt->log,"File loaded after test from cache: %s%s"LF,back[i].url_adr,back[i].url_fil); test_flush;
  3036.                         }
  3037.  
  3038.                         // finalize
  3039.                         //file_notify(back[i].url_adr, back[i].url_fil, back[i].url_sav, 0, 0, back[i].r.notmodified);        // not modified
  3040.                         if (back[i].r.statuscode>0) {
  3041.                           back_finalize(opt,cache,sback,i);
  3042.                         }
  3043.                         
  3044. #if DEBUGCA
  3045.                         printf("..document α jour aprΦs requΦte: %s%s\n",back[i].url_adr,back[i].url_fil);
  3046. #endif
  3047.                         
  3048.                         //printf(">%s status %d\n",back[p].r.contenttype,back[p].r.statuscode);
  3049.                       } else {  // erreur
  3050.                         back[i].status=0;  // terminΘ
  3051.                         back_set_finished(sback, i);
  3052.                         //printf("erreur cache\n");
  3053.                         
  3054.                       } 
  3055.                       
  3056. /********** NO - must complete the body! ********** */
  3057. #if 0
  3058.                     } else if (HTTP_IS_REDIRECT(back[i].r.statuscode)
  3059.                       || (back[i].r.statuscode==412)
  3060.                       || (back[i].r.statuscode==416)
  3061.                       ) {   // Ne pas prendre le html, erreurs connues et gΘrΘes
  3062. #if HTS_DEBUG_CLOSESOCK
  3063.                       DEBUG_W("back_wait(301,302,303,307,412,416..): deletehttp\n");
  3064. #endif
  3065.                       // Couper connexion
  3066.                       /*KA deletehttp(&back[i].r); back[i].r.soc=INVALID_SOCKET;*/
  3067.                       back_maydeletehttp(opt, cache, sback, i);
  3068.  
  3069.                       back[i].status=0;  // terminΘ
  3070.                       back_set_finished(sback, i);
  3071.                       // finalize
  3072.                       if (back[i].r.statuscode>0) {
  3073.                         back_finalize(opt,cache,sback,i);
  3074.                       }
  3075. #endif
  3076. /********** **************************** ********** */
  3077.                     } else {    // il faut aller le chercher
  3078.                       
  3079.                       // effacer buffer (requΦte)
  3080.                       if (!noFreebuff) {
  3081.                         deleteaddr(&back[i].r);
  3082.                         back[i].r.size=0;
  3083.                       }
  3084.                       
  3085.                       // traiter 206 (partial content)
  3086.                       // xxc SI CHUNK VERIFIER QUE CA MARCHE??
  3087.                       if (back[i].r.statuscode==206) {  // on nous envoie un morceau (la fin) coz une partie sur disque!
  3088.                         LLint sz=fsize(back[i].url_sav);
  3089. #if HDEBUG
  3090.                         printf("partial content: "LLintP" on disk..\n",(LLint)sz);
  3091. #endif
  3092.                         if (sz>=0) {
  3093.                           if (!is_hypertext_mime(back[i].r.contenttype, back[i].url_sav)) {    // pas HTML
  3094.                             if (opt->getmode&2) {    // on peut ecrire des non html  **sinon ben euhh sera interceptΘ plus loin, donc rap sur ce qui va sortir**
  3095.                               filenote(back[i].url_sav,NULL);    // noter fichier comme connu
  3096.                               file_notify(back[i].url_adr, back[i].url_fil, back[i].url_sav, 0, 1, back[i].r.notmodified);
  3097.                               back[i].r.out=fopen(fconv(back[i].url_sav),"ab");  // append
  3098.                               if (back[i].r.out) {
  3099.                                 back[i].r.is_write=1;    // Θcrire
  3100.                                 back[i].r.size=sz;    // dΘja Θcrit
  3101.                                 back[i].r.statuscode=200;  // Forcer 'OK'
  3102.                                 if (back[i].r.totalsize>0)
  3103.                                   back[i].r.totalsize+=sz;    // plus en fait
  3104.                                 fseek(back[i].r.out,0,SEEK_END);  // α la fin
  3105. #if HDEBUG
  3106.                                 printf("continue interrupted file\n");
  3107. #endif
  3108.                               } else {    // On est dans la m**
  3109.                                 back[i].status=0;  // terminΘ (voir plus loin)
  3110.                                 back_set_finished(sback, i);
  3111.                                 strcpybuff(back[i].r.msg,"Can not open partial file");
  3112.                               }
  3113.                             }
  3114.                           } else {    // mΘmoire
  3115.                             FILE* fp=fopen(fconv(back[i].url_sav),"rb");
  3116.                             if (fp) {
  3117.                               LLint alloc_mem=sz + 1;
  3118.                               if (back[i].r.totalsize>0)
  3119.                                 alloc_mem+=back[i].r.totalsize;            // AJOUTER RESTANT!
  3120.                               if ( deleteaddr(&back[i].r) && (back[i].r.adr=(char*) malloct((INTsys) alloc_mem)) ) {
  3121.                                 back[i].r.size=sz;
  3122.                                 if (back[i].r.totalsize>0)
  3123.                                   back[i].r.totalsize+=sz;    // plus en fait
  3124.                                 if (( fread(back[i].r.adr,1,(INTsys)sz,fp)) != sz) {
  3125.                                   back[i].status=0;  // terminΘ (voir plus loin)
  3126.                                   back_set_finished(sback, i);
  3127.                                   strcpybuff(back[i].r.msg,"Can not read partial file");
  3128.                                 } else {
  3129.                                   back[i].r.statuscode=200;  // Forcer 'OK'
  3130. #if HDEBUG
  3131.                                   printf("continue in mem interrupted file\n");
  3132. #endif
  3133.                                 }
  3134.                               } else {
  3135.                                 back[i].status=0;  // terminΘ (voir plus loin)
  3136.                                 back_set_finished(sback, i);
  3137.                                 strcpybuff(back[i].r.msg,"No memory for partial file");
  3138.                               }
  3139.                               fclose(fp);
  3140.                             } else {  // Argh.. 
  3141.                               back[i].status=0;  // terminΘ (voir plus loin)
  3142.                               back_set_finished(sback, i);
  3143.                               strcpybuff(back[i].r.msg,"Can not open partial file");
  3144.                             }
  3145.                           }
  3146.                         } else {    // Non trouvΘ??
  3147.                           back[i].status=0;  // terminΘ (voir plus loin)
  3148.                           back_set_finished(sback, i);
  3149.                           strcpybuff(back[i].r.msg,"Can not find partial file");
  3150.                         }
  3151.                         // Erreur?
  3152.                         if (back[i].status==0) {
  3153.                           if (back[i].r.soc!=INVALID_SOCKET) {
  3154. #if HTS_DEBUG_CLOSESOCK
  3155.                             DEBUG_W("back_wait(206 solve problems): deletehttp\n");
  3156. #endif
  3157.                             deletehttp(&back[i].r);
  3158.                           }
  3159.                           back[i].r.soc=INVALID_SOCKET;
  3160.                           //back[i].r.statuscode=206;  ????????
  3161.                           back[i].r.statuscode=STATUSCODE_NON_FATAL;
  3162.                           if (strnotempty(back[i].r.msg))
  3163.                             strcpybuff(back[i].r.msg,"Error attempting to solve status 206 (partial file)");
  3164.                         }
  3165.                       }
  3166.                       
  3167.                       if (back[i].status!=0) {  // non terminΘ (erreur)
  3168.                         if (!back[i].testmode) {    // fichier normal
  3169.                           
  3170.                           if (back[i].r.empty /* ?? && back[i].r.statuscode==200 */) {  // empty response
  3171.                             // Couper connexion
  3172.                             back_maydeletehttp(opt, cache, sback, i);
  3173.                             /* KA deletehttp(&back[i].r); back[i].r.soc=INVALID_SOCKET; */
  3174.                             back[i].status=0;  // terminΘ
  3175.                             back_set_finished(sback, i);
  3176.                             if ( deleteaddr(&back[i].r) && (back[i].r.adr=(char*) malloct((INTsys) 2)) ) {
  3177.                               back[i].r.adr[0] = 0;
  3178.                             }
  3179.                             back_finalize(opt,cache,sback,i);
  3180.                           }
  3181.                           else if (!back[i].r.is_chunk) {    // pas de chunk
  3182.                             //if (back[i].r.http11!=2) {    // pas de chunk
  3183.                             back[i].is_chunk=0;
  3184.                             back[i].status=1;     // start body
  3185.                           } else {
  3186. #if CHUNKDEBUG==1
  3187.                             printf("[%d] chunk encoding detected %s..\n",(int)back[i].r.soc, back[i].url_fil);
  3188. #endif
  3189.                             back[i].is_chunk=1;
  3190.                             back[i].chunk_adr=NULL;
  3191.                             back[i].chunk_size=0;
  3192.                             back[i].chunk_blocksize=0;
  3193.                             back[i].status=98;    // start body wait chunk
  3194.                             back[i].r.totalsize=0;   /* devalidate size! (rfc) */
  3195.                           }
  3196.                           if (back[i].rateout>0) {
  3197.                             back[i].rateout_time=time_local();  // refresh pour transfer rate
  3198.                           }
  3199. #if HDEBUG
  3200.                           printf("(buffer) start body!\n");
  3201. #endif
  3202.                         } else {     // mode test, ne pas passer en 1!!
  3203.                           back[i].status=0;    // READY
  3204.                           back_set_finished(sback, i);
  3205. #if HTS_DEBUG_CLOSESOCK
  3206.                           DEBUG_W("back_wait(test ok): deletehttp\n");
  3207. #endif
  3208.                           deletehttp(&back[i].r); back[i].r.soc=INVALID_SOCKET;
  3209.                           if (back[i].r.statuscode==200) {
  3210.                             strcpybuff(back[i].r.msg,"Test: OK");
  3211.                             back[i].r.statuscode=STATUSCODE_TEST_OK;    // test rΘussi
  3212.                           }
  3213.                           else {    // test a ΘchouΘ, on ne change rien sauf que l'erreur est α titre indicatif
  3214.                             char tempo[1000];
  3215.                             strcpybuff(tempo,back[i].r.msg);
  3216.                             strcpybuff(back[i].r.msg,"Test: ");
  3217.                             strcatbuff(back[i].r.msg,tempo);
  3218.                           }
  3219.                           
  3220.                         }
  3221.                       }
  3222.                       
  3223.                       } 
  3224.                       
  3225.                       /*}*/
  3226.                       
  3227.                   }  // si LF
  3228.                 }  // r.size>2
  3229.               }  // si == 99
  3230.               
  3231.             } // si pas d'erreurs
  3232. #if BDEBUG==1
  3233.             printf("bytes overall: %d\n",back[i].r.size);
  3234. #endif
  3235.           }  // donnΘes dispo
  3236.           
  3237.           // en cas d'erreur cl, supprimer Θventuel fichier sur disque
  3238. #if HTS_REMOVE_BAD_FILES
  3239.           if (back[i].status<0) {
  3240.             if (!back[i].testmode) {    // pas en test
  3241.               unlink(back[i].url_sav);    // Θliminer fichier (endommagΘ)
  3242.               //printf("&& %s\n",back[i].url_sav);
  3243.             }
  3244.           }
  3245. #endif
  3246.  
  3247.           /* funny log for commandline users */
  3248.           //if (!opt->quiet) {  
  3249.           // petite animation
  3250.           if (opt->verbosedisplay==1) {
  3251.             if (back[i].status==0) {
  3252.               if (back[i].r.statuscode==200)
  3253.                 printf("* %s%s ("LLintP" bytes) - OK"VT_CLREOL"\r",back[i].url_adr,back[i].url_fil,(LLint)back[i].r.size);
  3254.               else
  3255.                 printf("* %s%s ("LLintP" bytes) - %d"VT_CLREOL"\r",back[i].url_adr,back[i].url_fil,(LLint)back[i].r.size,back[i].r.statuscode);
  3256.               fflush(stdout);
  3257.             }
  3258.           }
  3259.           //}
  3260.           
  3261.  
  3262.       } // status>0
  3263.     }  // for
  3264.     
  3265.     // vΘrifier timeouts
  3266.     if (gestion_timeout) {
  3267.       TStamp act;
  3268.       act=time_local();    // temps en secondes
  3269.       for(i_mod = 0 ; i_mod < (unsigned int) back_max ; i_mod++) {
  3270.       // for(i=0;i<back_max;i++) {
  3271.         unsigned int i = ( i_mod + mod_random ) % ( back_max );     
  3272.         if (back[i].status>0) {  // rΘception/connexion/..
  3273.           if (back[i].timeout>0) {
  3274.             //printf("time check %d\n",((int) (act-back[i].timeout_refresh))-back[i].timeout);
  3275.             if (((int) (act-back[i].timeout_refresh))>=back[i].timeout) {
  3276.               if (back[i].r.soc!=INVALID_SOCKET) {
  3277. #if HTS_DEBUG_CLOSESOCK
  3278.                 DEBUG_W("back_wait(timeout): deletehttp\n");
  3279. #endif
  3280.                 deletehttp(&back[i].r);
  3281.               }
  3282.               back[i].r.soc=INVALID_SOCKET;
  3283.               back[i].r.statuscode=STATUSCODE_TIMEOUT;
  3284.               if (back[i].status==100)
  3285.                 strcpybuff(back[i].r.msg,"Connect Time Out");
  3286.               else if (back[i].status==101)
  3287.                 strcpybuff(back[i].r.msg,"DNS Time Out");
  3288.               else
  3289.                 strcpybuff(back[i].r.msg,"Receive Time Out");
  3290.               back[i].status=0;  // terminΘ
  3291.               back_set_finished(sback, i);
  3292.             } else if ((back[i].rateout>0) && (back[i].status<99)) {
  3293.               if (((int) (act-back[i].rateout_time))>=HTS_WATCHRATE) {   // checker au bout de 15s
  3294.                 if ( (int) ((back[i].r.size)/(act-back[i].rateout_time)) < back[i].rateout ) {  // trop lent
  3295.                   back[i].status=0;  // terminΘ
  3296.                   back_set_finished(sback, i);
  3297.                   if (back[i].r.soc!=INVALID_SOCKET) {
  3298. #if HTS_DEBUG_CLOSESOCK
  3299.                     DEBUG_W("back_wait(rateout): deletehttp\n");
  3300. #endif
  3301.                     deletehttp(&back[i].r);
  3302.                   }
  3303.                   back[i].r.soc=INVALID_SOCKET;
  3304.                   back[i].r.statuscode=STATUSCODE_SLOW;
  3305.                   strcpybuff(back[i].r.msg,"Transfer Rate Too Low");
  3306.                 }
  3307.               }
  3308.             }
  3309.           }
  3310.         }
  3311.       }
  3312.     }
  3313.     max_loop--;
  3314. #if HTS_ANALYSTE
  3315.     max_loop_chk++;
  3316. #endif
  3317.   } while((busy_state) && (busy_recv) && (max_loop>0));
  3318. #if HTS_ANALYSTE
  3319.   if ((!busy_recv) && (!busy_state)) {
  3320.     if (max_loop_chk>=1) {
  3321.       Sleep(10);    // un tite pause pour Θviter les lag..
  3322.     }
  3323.   }
  3324. #endif
  3325. }
  3326.  
  3327. int back_checksize(httrackp* opt,lien_back* eback,int check_only_totalsize) {
  3328.   LLint size_to_test;
  3329.   if (check_only_totalsize)
  3330.     size_to_test=eback->r.totalsize;
  3331.   else
  3332.     size_to_test=max(eback->r.totalsize,eback->r.size);
  3333.   if (size_to_test>=0) {
  3334.     
  3335.     /* Interdiction taille par le wizard? */
  3336.     if (hts_testlinksize(opt,eback->url_adr,eback->url_fil,eback->r.totalsize/1024)==-1) {
  3337.       return 0;     /* interdit */
  3338.     }                     
  3339.     
  3340.     /* vΘrifier taille classique (heml et non html) */
  3341.     if ((istoobig(size_to_test,eback->maxfile_html,eback->maxfile_nonhtml,eback->r.contenttype))) {
  3342.       return 0;     /* interdit */
  3343.     }
  3344.   }
  3345.   return 1;
  3346. }
  3347.  
  3348. int back_checkmirror(httrackp* opt) {
  3349.   // Check max time
  3350.   if ((opt->maxsite>0) && (HTS_STAT.stat_bytes >= opt->maxsite)) {
  3351.     if (opt->errlog) {
  3352.       fprintf(opt->errlog,"More than "LLintP" bytes have been transfered.. giving up"LF,(LLint)opt->maxsite);
  3353.       test_flush;
  3354.     } 
  3355.     return 0;
  3356.   } else if ((opt->maxtime>0) && ((time_local()-HTS_STAT.stat_timestart)>opt->maxtime)) {            
  3357.     if (opt->errlog) {
  3358.       fprintf(opt->errlog,"More than %d seconds passed.. giving up"LF,opt->maxtime);
  3359.       test_flush;
  3360.     } 
  3361.     return 0;
  3362.   }
  3363.   return 1;   /* Ok, go on */
  3364. }
  3365.  
  3366.  
  3367. // octets transfΘrΘs + add
  3368. LLint back_transfered(LLint nb,struct_back* sback) {
  3369.   lien_back* const back = sback->lnk;
  3370.   const int back_max = sback->count;
  3371.   int i;
  3372.   // ajouter octets en instance
  3373.   for(i=0;i<back_max;i++)
  3374.     if ((back[i].status>0) && (back[i].status<99 || back[i].status>=1000))
  3375.       nb+=back[i].r.size;
  3376.   // stored (ready) slots
  3377.   if (sback->ready != NULL) {
  3378.     struct_inthash_enum e = inthash_enum_new((inthash)sback->ready);
  3379.     inthash_chain* item;
  3380.     while((item = inthash_enum_next(&e))) {
  3381.       lien_back* ritem = (lien_back*) item->value.ptr;
  3382.       if ((ritem->status>0) && (ritem->status<99 || ritem->status>=1000))
  3383.         nb+=ritem->r.size;
  3384.     }
  3385.   }
  3386.   return nb;      
  3387. }
  3388.  
  3389. // infos backing
  3390. // j: 1 afficher sockets 2 afficher autres 3 tout afficher
  3391. void back_info(struct_back* sback,int i,int j,FILE* fp) {
  3392.   lien_back* const back = sback->lnk;
  3393.   const int back_max = sback->count;
  3394.   assertf(i >= 0 && i < back_max);
  3395.   if (back[i].status>=0) {
  3396.     char BIGSTK s[HTS_URLMAXSIZE*2+1024]; 
  3397.     s[0]='\0';
  3398.     back_infostr(sback,i,j,s);
  3399.     strcatbuff(s,LF);
  3400.     fprintf(fp,"%s",s);
  3401.   }
  3402. }
  3403.  
  3404. // infos backing
  3405. // j: 1 afficher sockets 2 afficher autres 3 tout afficher
  3406. void back_infostr(struct_back* sback,int i,int j,char* s) {
  3407.   lien_back* const back = sback->lnk;
  3408.   const int back_max = sback->count;
  3409.   assertf(i >= 0 && i < back_max);
  3410.   if (back[i].status>=0) {
  3411.     int aff=0;
  3412.     if (j & 1) {
  3413.       if (back[i].status==100) {
  3414.         strcatbuff(s,"CONNECT ");
  3415.       } else if (back[i].status==99) {
  3416.         strcatbuff(s,"INFOS ");
  3417.         aff=1;
  3418.       } else if (back[i].status==98 || back[i].status==97) {
  3419.         strcatbuff(s,"INFOSC");             // infos chunk
  3420.         aff=1;
  3421.       }
  3422.       else if (back[i].status>0) {
  3423. #if HTS_ANALYSTE==2
  3424.         strcatbuff(s,"WAIT ");
  3425. #else
  3426.         strcatbuff(s,"RECEIVE "); 
  3427. #endif
  3428.         aff=1; 
  3429.       }
  3430.     } 
  3431.     if (j & 2) {
  3432.       if (back[i].status==0) {
  3433.         switch (back[i].r.statuscode) {
  3434.         case 200:
  3435.           strcatbuff(s,"READY ");
  3436.           aff=1;
  3437.           break;
  3438. #if HTS_ANALYSTE==2
  3439.         default:
  3440.           strcatbuff(s,"ERROR ");
  3441.           break;
  3442. #else
  3443.         case -1:
  3444.           strcatbuff(s,"ERROR ");
  3445.           aff=1;
  3446.           break;
  3447.         case -2:
  3448.           strcatbuff(s,"TIMEOUT ");
  3449.           aff=1;
  3450.           break;
  3451.         case -3:
  3452.           strcatbuff(s,"TOOSLOW ");
  3453.           aff=1;
  3454.           break;
  3455.         case 400:
  3456.           strcatbuff(s,"BADREQUEST ");
  3457.           aff=1;
  3458.           break;
  3459.         case 401: case 403:
  3460.           strcatbuff(s,"FORBIDDEN ");
  3461.           aff=1;
  3462.           break;
  3463.         case 404:
  3464.           strcatbuff(s,"NOT FOUND ");
  3465.           aff=1;
  3466.           break;
  3467.         case 500:
  3468.           strcatbuff(s,"SERVERROR ");
  3469.           aff=1;
  3470.           break;
  3471.         default:
  3472.           {
  3473.             char s2[256];
  3474.             sprintf(s2,"ERROR(%d)",back[i].r.statuscode);
  3475.             strcatbuff(s,s2);
  3476.           }
  3477.           aff=1;
  3478. #endif
  3479.         }
  3480.       }
  3481.     }
  3482.     
  3483.     if (aff) {
  3484.       {
  3485.         char BIGSTK s2[HTS_URLMAXSIZE*2+1024];
  3486.         sprintf(s2,"\"%s",back[i].url_adr); strcatbuff(s,s2);
  3487.         
  3488.         if (back[i].url_fil[0]!='/') strcatbuff(s,"/");
  3489.         sprintf(s2,"%s\" ",back[i].url_fil); strcatbuff(s,s2);
  3490.         sprintf(s,LLintP" "LLintP" ",(LLint)back[i].r.size,(LLint)back[i].r.totalsize); strcatbuff(s,s2);
  3491.       }
  3492.     }
  3493.   }
  3494. }
  3495.  
  3496. // -- backing --
  3497.  
  3498. #undef test_flush
  3499.